Skip to content

Commit

Permalink
Refactor buildmetadata to not require svn export
Browse files Browse the repository at this point in the history
  • Loading branch information
@rowanseymour
rowanseymour committed Dec 14, 2023
1 parent 08d3fe1 commit 4c9367a
Show file tree
Hide file tree
Showing 4 changed files with 182 additions and 162 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,4 @@ geocoding
buildmetadata
functions/*
dist/
_build
287 changes: 153 additions & 134 deletions cmd/buildmetadata/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ import (
"io/ioutil"
"log"
"math"
"net/http"
"os"
"os/exec"
"path/filepath"
Expand All @@ -24,28 +23,157 @@ import (
"github.com/nyaruka/phonenumbers"
)

func main() {
if err := buildMetadata(); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}

func buildMetadata() error {
fmt.Print("Cloning upstream repo... ")

if err := cloneUpstreamRepo("git@github.com:google/libphonenumber.git"); err != nil {
return err
}

fmt.Print("OK\nBuilding number metadata...")

metadata, err := buildNumberMetadata("resources/PhoneNumberMetadata.xml", "metadataData", "metadata_bin.go", false)
if err != nil {
return err
}

fmt.Print("OK\nBuilding short number metadata...")

_, err = buildNumberMetadata("resources/ShortNumberMetadata.xml", "shortNumberMetadataData", "shortnumber_metadata_bin.go", true)
if err != nil {
return err
}

fmt.Print("OK\nBuilding region metadata...")

if err := buildRegionMetadata(metadata, "regionMapData", "countrycode_to_region_bin.go"); err != nil {
return err
}

fmt.Print("OK\nBuilding timezone metadata...")

if err := buildTimezoneMetadata("resources/timezones/map_data.txt", "timezoneMapData", "prefix_to_timezone_bin.go"); err != nil {
return err
}

buildPrefixData(&carrier)
buildPrefixData(&geocoding)

return nil
}

func cloneUpstreamRepo(url string) error {
os.RemoveAll("_build")

cmd := exec.Command("git", "clone", "--depth=1", url, "_build")
if err := cmd.Run(); err != nil {
return fmt.Errorf("error cloning upstream repo: %w", err)
}

return nil
}

func buildNumberMetadata(srcFile, varName, dstFile string, short bool) (*phonenumbers.PhoneMetadataCollection, error) {
body, err := os.ReadFile("_build/" + srcFile)
if err != nil {
return nil, fmt.Errorf("error reading %s: %w", srcFile, err)
}

collection, err := phonenumbers.BuildPhoneMetadataCollection(body, false, false, short)
if err != nil {
return nil, fmt.Errorf("error parsing %s: %w", srcFile, err)
}

data, err := proto.Marshal(collection)
if err != nil {
return nil, fmt.Errorf("error marshaling metadata as protobuf: %w", err)
}

if err := os.WriteFile(dstFile, generateBinFile(varName, data), os.FileMode(0664)); err != nil {
return nil, fmt.Errorf("error writing %s: %w", dstFile, err)
}

return collection, nil
}

func buildRegionMetadata(metadata *phonenumbers.PhoneMetadataCollection, varName, dstFile string) error {
regionMap := phonenumbers.BuildCountryCodeToRegionMap(metadata)

// generate our map data
data, err := renderMap(dstFile, regionMap)
if err != nil {
return fmt.Errorf("error generating %s: %w", dstFile, err)
}

if err := os.WriteFile(dstFile, generateBinFile(varName, data), os.FileMode(0664)); err != nil {
return fmt.Errorf("error writing %s: %w", dstFile, err)
}

return nil
}

func buildTimezoneMetadata(srcFile, varName, dstFile string) error {
body, err := os.ReadFile("_build/" + srcFile)
if err != nil {
return fmt.Errorf("error reading %s: %w", srcFile, err)
}

// build our map of prefix to timezones
prefixMap := make(map[int][]string)
for _, line := range strings.Split(string(body), "\n") {
if strings.HasPrefix(line, "#") {
continue
}

if strings.TrimSpace(line) == "" {
continue
}

fields := strings.Split(line, "|")
if len(fields) != 2 {
return fmt.Errorf("invalid format in timezone file: %s", line)
}

zones := strings.Split(fields[1], "&")
if len(zones) < 1 {
return fmt.Errorf("invalid format in timezone file: %s", line)
}

// parse our prefix
prefix, err := strconv.Atoi(fields[0])
if err != nil {
return fmt.Errorf("invalid prefix in line: %s", line)
}
prefixMap[prefix] = zones
}

// generate our map data
data, err := renderMap(dstFile, prefixMap)
if err != nil {
return fmt.Errorf("error generating %s: %w", dstFile, err)
}

if err := os.WriteFile(dstFile, generateBinFile(varName, data), os.FileMode(0664)); err != nil {
return fmt.Errorf("error writing %s: %w", dstFile, err)
}

return nil
}

type prefixBuild struct {
url string
dir string
srcPath string
varName string
}

const (
metadataURL = "https://raw.githubusercontent.com/googlei18n/libphonenumber/master/resources/PhoneNumberMetadata.xml"
metadataPath = "metadata_bin.go"

shortNumberMetadataURL = "https://raw.githubusercontent.com/googlei18n/libphonenumber/master/resources/ShortNumberMetadata.xml"
shortNumberMetadataPath = "shortnumber_metadata_bin.go"

tzURL = "https://raw.githubusercontent.com/googlei18n/libphonenumber/master/resources/timezones/map_data.txt"
tzPath = "prefix_to_timezone_bin.go"
tzVar = "timezoneMapData"

regionPath = "countrycode_to_region_bin.go"
regionVar = "regionMapData"
)

var carrier = prefixBuild{
url: "https://github.com/googlei18n/libphonenumber/trunk/resources/carrier",
dir: "carrier",
Expand All @@ -60,20 +188,6 @@ var geocoding = prefixBuild{
varName: "geocodingMapData",
}

func fetchURL(url string) []byte {
resp, err := http.Get(url)
if err != nil || resp.StatusCode != 200 {
log.Fatalf("Error fetching URL '%s': %s", url, err)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatalf("Error reading body: %s", err)
}

return body
}

func svnExport(dir string, url string) {
os.RemoveAll(dir)
cmd := exec.Command(
Expand Down Expand Up @@ -122,56 +236,13 @@ func writeFile(filePath string, data []byte) {
}

fmt.Printf("Writing new %s\n", filePath)
err := ioutil.WriteFile(filePath, data, os.FileMode(0664))
err := os.WriteFile(filePath, data, os.FileMode(0664))
if err != nil {
log.Fatalf("Error writing '%s': %s", filePath, err)
}
}

func buildRegions(metadata *phonenumbers.PhoneMetadataCollection) {
log.Println("Building region map")
regionMap := phonenumbers.BuildCountryCodeToRegionMap(metadata)
writeIntStringArrayMap(regionPath, regionVar, regionMap)
}

func buildTimezones() {
log.Println("Building timezone map")
body := fetchURL(tzURL)

// build our map of prefix to timezones
prefixMap := make(map[int][]string)
for _, line := range strings.Split(string(body), "\n") {
if strings.HasPrefix(line, "#") {
continue
}

if strings.TrimSpace(line) == "" {
continue
}

fields := strings.Split(line, "|")
if len(fields) != 2 {
log.Fatalf("Invalid format in timezone file: %s", line)
}

zones := strings.Split(fields[1], "&")
if len(zones) < 1 {
log.Fatalf("Invalid format in timezone file: %s", line)
}

// parse our prefix
prefix, err := strconv.Atoi(fields[0])
if err != nil {
log.Fatalf("Invalid prefix in line: %s", line)
}
prefixMap[prefix] = zones
}

// then write our file
writeIntStringArrayMap(tzPath, tzVar, prefixMap)
}

func writeIntStringArrayMap(path string, varName string, prefixMap map[int][]string) {
func renderMap(path string, prefixMap map[int][]string) ([]byte, error) {
// build lists of our keys and values
keys := make([]int, 0, len(prefixMap))
values := make([]string, 0, 255)
Expand Down Expand Up @@ -200,15 +271,15 @@ func writeIntStringArrayMap(path string, varName string, prefixMap map[int][]str
// first write our values, as length of string and raw bytes
joinedValues := strings.Join(values, "\n")
if err := binary.Write(data, binary.LittleEndian, uint32(len(joinedValues))); err != nil {
log.Fatal(err)
return nil, err
}
if err := binary.Write(data, binary.LittleEndian, []byte(joinedValues)); err != nil {
log.Fatal(err)
return nil, err
}

// then the number of keys
if err := binary.Write(data, binary.LittleEndian, uint32(len(keys))); err != nil {
log.Fatal(err)
return nil, err
}

// we write our key / value pairs as a varint of the difference of the previous prefix
Expand All @@ -220,72 +291,29 @@ func writeIntStringArrayMap(path string, varName string, prefixMap map[int][]str
diff := key - last
l := binary.PutUvarint(intBuf, uint64(diff))
if err := binary.Write(data, binary.LittleEndian, intBuf[:l]); err != nil {
log.Fatal(err)
return nil, err
}

// then our values
values := prefixMap[key]

// write our number of values
if err := binary.Write(data, binary.LittleEndian, uint8(len(values))); err != nil {
log.Fatal(err)
return nil, err
}

// then each value as the interned index
for _, v := range values {
valueIntern := internMap[v]
if err := binary.Write(data, binary.LittleEndian, uint16(valueIntern)); err != nil {
log.Fatal(err)
return nil, err
}
}

last = key
}

// then write our file
writeFile(path, generateBinFile(varName, data.Bytes()))
}

func buildMetadata() *phonenumbers.PhoneMetadataCollection {
log.Println("Fetching PhoneNumberMetadata.xml from Github")
body := fetchURL(metadataURL)

log.Println("Building new metadata collection")
collection, err := phonenumbers.BuildPhoneMetadataCollection(body, false, false, false)
if err != nil {
log.Fatalf("Error converting XML: %s", err)
}

// write it out as a protobuf
data, err := proto.Marshal(collection)
if err != nil {
log.Fatalf("Error marshalling metadata: %v", err)
}

log.Println("Writing new metadata_bin.go")
writeFile(metadataPath, generateBinFile("metadataData", data))
return collection
}

func buildShortNumberMetadata() *phonenumbers.PhoneMetadataCollection {
log.Println("Fetching ShortNumberMetadata.xml from Github")
body := fetchURL(shortNumberMetadataURL)

log.Println("Building new short number metadata collection")
collection, err := phonenumbers.BuildPhoneMetadataCollection(body, false, false, true)
if err != nil {
log.Fatalf("Error converting XML: %s", err)
}

// write it out as a protobuf
data, err := proto.Marshal(collection)
if err != nil {
log.Fatalf("Error marshalling metadata: %v", err)
}

log.Println("Writing new metadata_bin.go")
writeFile(shortNumberMetadataPath, generateBinFile("shortNumberMetadataData", data))
return collection
return data.Bytes(), nil
}

// generates the file contents for a data file
Expand Down Expand Up @@ -476,12 +504,3 @@ func readMappingsForDir(dir string) map[int]string {
log.Printf("Read %d mappings in %s\n", len(mappings), dir)
return mappings
}

func main() {
metadata := buildMetadata()
buildShortNumberMetadata()
buildRegions(metadata)
buildTimezones()
buildPrefixData(&carrier)
buildPrefixData(&geocoding)
}
10 changes: 5 additions & 5 deletions prefix_to_carriers_bin.go
View file

Large diffs are not rendered by default.

46 changes: 23 additions & 23 deletions prefix_to_geocodings_bin.go
View file

Large diffs are not rendered by default.

0 comments on commit 4c9367a

Please sign in to comment.