parser: support entire index format

.mise.toml

@@ -5,6 +5,7 @@ golangci-lint = "1.56.2"
 "go:golang.org/x/tools/cmd/goimports" = "latest"
 
 [tasks.tests]
+alias = "test"
 description = "Run tests"
 run = ["gotestsum", "golangci-lint run --allow-parallel-runners --fast"]
 

cmd/binhost/binhost.go

@@ -17,7 +17,32 @@
 // README in the root of the repository for more information.
 package main
 
+import (
+	"fmt"
+	"net/http"
+	"os"
+
+	"github.com/jaredallard/binhost/internal/parser"
+)
+
 // main runs the binhost server.
 func main() {
+	req, err := http.NewRequest("GET", "https://gentoo.rgst.io/t/arm64/asahi/Packages", nil)
+	if err != nil {
+		panic(err)
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		panic(err)
+	}
+
+	defer resp.Body.Close()
+
+	pkgs, err := parser.ParsePackages(resp.Body)
+	if err != nil {
+		panic(fmt.Errorf("failed to parse packages: %w", err))
+	}
 
+	pkgs.EncodeInto(os.Stdout)
 }

internal/parser/parser.go

@@ -18,69 +18,54 @@
 package parser
 
 import (
-	"bufio"
-	"fmt"
 	"io"
-	"reflect"
-	"strconv"
-	"strings"
-
-	"github.com/davecgh/go-spew/spew"
 )
 
-// parseColonDocuments parses fields from a colon separated file and
-// returns the documents as a slice of maps. Empty newlines are
-// considered to be the end of a document.
-func parseColonDocuments(r io.Reader) ([]map[string]string, error) {
-	out := make([]map[string]string, 0)
-	cur := make(map[string]string)
-
-	scanner := bufio.NewScanner(r)
-	for scanner.Scan() {
-		line := scanner.Text()
-
-		// if the line is empty, we've reached the end of the document
-		if line == "" {
-			out = append(out, cur)
-			cur = make(map[string]string)
-			continue
-		}
-
-		// split the line by the first colon
-		parts := strings.SplitN(line, ": ", 2)
-		if len(parts) != 2 {
-			return nil, fmt.Errorf("invalid line: %s", line)
-		}
-
-		// set the key and value
-		cur[parts[0]] = parts[1]
-	}
-	if scanner.Err() != nil {
-		return nil, scanner.Err()
-	}
-
-	// If we ended with a non-empty document, append it to the output now.
-	if len(cur) > 0 {
-		out = append(out, cur)
-	}
-	return out, nil
+// Index is a binhost index file (Packages).
+type Index struct {
+	AcceptLicense         string   `colon:"ACCEPT_LICENSE"`
+	AcceptProperties      string   `colon:"ACCEPT_PROPERTIES"`
+	AcceptRestrict        string   `colon:"ACCEPT_RESTRICT"`
+	AcceptKeywords        []string `colon:"ACCEPT_KEYWORDS"`
+	Arch                  string   `colon:"ARCH"`
+	CBuild                string   `colon:"CBUILD"`
+	CHost                 string   `colon:"CHOST"`
+	ConfigProtect         []string `colon:"CONFIG_PROTECT"`
+	ConfigProtectMask     []string `colon:"CONFIG_PROTECT_MASK"`
+	ELibc                 string   `colon:"ELIBC"`
+	Features              []string `colon:"FEATURES"`
+	GentooMirrors         string   `colon:"GENTOO_MIRRORS"`
+	IUseImplicit          string   `colon:"IUSE_IMPLICIT"`
+	Kernel                string   `colon:"KERNEL"`
+	Packages              int      `colon:"PACKAGES"`
+	Profile               string   `colon:"PROFILE"`
+	Timestamp             int      `colon:"TIMESTAMP"`
+	Use                   string   `colon:"USE"`
+	UseExpand             []string `colon:"USE_EXPAND"`
+	UseExpandHidden       []string `colon:"USE_EXPAND_HIDDEN"`
+	UseExpandImplicit     []string `colon:"USE_EXPAND_IMPLICIT"`
+	UseExpandUnprefixed   []string `colon:"USE_EXPAND_UNPREFIXED"`
+	UseExpandValuesArch   []string `colon:"USE_EXPAND_VALUES_ARCH"`
+	UseExpandValuesELibc  []string `colon:"USE_EXPAND_VALUES_ELIBC"`
+	UseExpandValuesKernel []string `colon:"USE_EXPAND_VALUES_KERNEL"`
+	Version               int      `colon:"VERSION"`
+
+	// PackageEntries is a slice of packages contained within the index.
+	PackageEntries []Package
 }
 
-// Packages is a Gentoo binhost packages file.
-type Packages []Package
-
 // EncodeInto serializes the packages into the given writer using the
 // standard Packages format.
 //
 // See: https://wiki.gentoo.org/wiki/Binary_package_guide
-func (pkgs Packages) EncodeInto(w io.Writer) error {
-	for _, pkg := range pkgs {
-		if err := pkg.EncodeInto(w); err != nil {
-			return err
-		}
+func (index Index) EncodeInto(w io.Writer) error {
+	// Write the header first.
+	if err := encodeColonFormat(w, &index); err != nil {
+		return err
+	}
 
-		// Write a newline to separate the packages
-		if _, err := w.Write([]byte("\n")); err != nil {
+	for _, pkg := range index.PackageEntries {
+		if err := pkg.EncodeInto(w); err != nil {
 			return err
 		}
 	}
@@ -94,119 +79,67 @@ type Package struct {
 	BuildID       string   `colon:"BUILD_ID"`
 	BuildTime     string   `colon:"BUILD_TIME"`
 	CPV           string   `colon:"CPV"`
-	DefinedPhases string   `colon:"DEFINED_PHASES"`
-	Depends       string   `colon:"DEPEND"`
+	DefinedPhases []string `colon:"DEFINED_PHASES"`
 	EAPI          int      `colon:"EAPI"`
+	ELibc         string   `colon:"ELIBC"`
 	IUse          string   `colon:"IUSE"`
+	Use           string   `colon:"USE"`
 	Keywords      []string `colon:"KEYWORDS"`
 	Licenses      []string `colon:"LICENSE"`
 	Path          string   `colon:"PATH"`
+	Slot          string   `colon:"SLOT"`
+	Depends       string   `colon:"DEPEND"`
+	IDepend       string   `colon:"IDEPEND"`
+	PDepends      string   `colon:"PDEPEND"`
 	RDepends      string   `colon:"RDEPEND"`
-	Requires      string   `colon:"REQUIRES"`
+	Requires      []string `colon:"REQUIRES"`
+	Restrict      string   `colon:"RESTRICT"`
+	Provides      []string `colon:"PROVIDES"`
 	SHA1          string   `colon:"SHA1"`
+	MD5           string   `colon:"MD5"`
 	Size          int      `colon:"SIZE"`
-	Use           string   `colon:"USE"`
 	ModifiedTime  int      `colon:"MTIME"`
 	Repo          string   `colon:"REPO"`
 }
 
 // EncodeInto serializes the package into the given writer using the
 // standard Package format.
 func (pkg *Package) EncodeInto(w io.Writer) error {
-	prv := reflect.ValueOf(pkg).Elem()
-	prt := prv.Type()
-
-	for i := 0; i < prv.NumField(); i++ {
-		fv := prv.Field(i)
-		ft := prt.Field(i)
-
-		value := fv.Interface()
-		if value == nil || fv.IsZero() {
-			continue
-		}
-
-		key := ft.Tag.Get("colon")
-		if key == "" {
-			// default to the field name as it is in the struct
-			key = ft.Name
-		}
-		switch v := value.(type) {
-		case string:
-			value = v
-		case []string:
-			value = strings.Join(v, " ")
-		default:
-			value = fmt.Sprintf("%v", v)
-		}
-
-		if _, err := w.Write([]byte(fmt.Sprintf("%s: %s\n", key, value))); err != nil {
-			return err
-		}
-	}
-
-	return nil
+	return encodeColonFormat(w, pkg)
 }
 
-// ParsePackages parses the provided reader into a slice of packages.
-func ParsePackages(r io.Reader) (Packages, error) {
+// ParsePackages parses the provided reader into a the Index type.
+func ParsePackages(r io.Reader) (*Index, error) {
 	docs, err := parseColonDocuments(r)
 	if err != nil {
 		return nil, err
 	}
 
-	spew.Dump(docs)
+	// Read the first document into the index.
+	if len(docs) == 0 {
+		return nil, nil
+	}
 
-	pkgs := make(Packages, 0, len(docs))
-	for _, doc := range docs {
-		var pkg Package
-		pkgT := reflect.TypeOf(pkg)
-		pkgV := reflect.ValueOf(&pkg).Elem()
-
-		// Create a map of the struct tags to the field index
-		tagToField := make(map[string]int)
-		for i := 0; i < pkgT.NumField(); i++ {
-			tag := pkgT.Field(i).Tag.Get("colon")
-			if tag == "" {
-				tag = pkgT.Field(i).Name
-			}
-			tagToField[tag] = i
-		}
+	var index Index
+	if err := decodeColonFormat(docs[0], &index); err != nil {
+		return nil, err
+	}
 
-		// Set the fields from the document
-		for k, v := range doc {
-			fieldIndex, ok := tagToField[k]
-			if !ok {
-				return nil, fmt.Errorf("unknown field: %s", k)
-			}
-
-			field := pkgV.Field(fieldIndex)
-			switch field.Kind() {
-			case reflect.String:
-				field.SetString(v)
-			case reflect.Slice:
-				field.Set(reflect.ValueOf(strings.Fields(v)))
-			case reflect.Int:
-				i, err := strconv.Atoi(v)
-				if err != nil {
-					return nil, fmt.Errorf("invalid integer: %s", v)
-				}
-				field.SetInt(int64(i))
-			default:
-				return nil, fmt.Errorf("unsupported field type: %s", field.Kind())
-			}
-
-			// Remove the field from the map so we can check for missing
-			// fields later.
-			delete(doc, k)
-		}
+	// No package entries, return the index.
+	if len(docs) == 1 {
+		return &index, nil
+	}
 
-		// Check for missing fields
-		if len(doc) > 0 {
-			return nil, fmt.Errorf("unknown fields: %v", doc)
-		}
+	index.PackageEntries = make([]Package, len(docs)-1)
 
-		pkgs = append(pkgs, pkg)
+	// Read the rest of the documents into the package entries.
+	for i, doc := range docs[1:] {
+		var pkg Package
+		if err := decodeColonFormat(doc, &pkg); err != nil {
+			return nil, err
+		}
+		index.PackageEntries[i] = pkg
 	}
 
-	return pkgs, nil
+	return &index, nil
 }

internal/parser/parser_test.go

@@ -16,15 +16,30 @@ func TestCanEncodeAPackage(t *testing.T) {
 
 	var buf bytes.Buffer
 	assert.NilError(t, pkg.EncodeInto(&buf))
-	assert.Equal(t, "CPV: x11-terms/alacritty-0.12.3\n", buf.String())
+	assert.Equal(t, "CPV: x11-terms/alacritty-0.12.3\n\n", buf.String())
 }
 
-// TODO: Implement this.
 func TestCanReadPackages(t *testing.T) {
-	input := `CPV: x11-terms/alacritty-0.12.3` + "\n"
+	input := `ARCH: arm64` + "\n" + "\n" + `CPV: x11-terms/alacritty-0.12.3` + "\n"
 
-	pkgs, err := parser.ParsePackages(strings.NewReader(input))
+	index, err := parser.ParsePackages(strings.NewReader(input))
 	assert.NilError(t, err)
-	assert.Equal(t, 1, len(pkgs))
-	assert.Equal(t, "x11-terms/alacritty-0.12.3", pkgs[0].CPV)
+	assert.Equal(t, 1, len(index.PackageEntries))
+	assert.Equal(t, "arm64", index.Arch)
+	assert.Equal(t, "x11-terms/alacritty-0.12.3", index.PackageEntries[0].CPV)
+}
+
+func TestCanEncodePackages(t *testing.T) {
+	index := parser.Index{
+		Arch: "arm64",
+		PackageEntries: []parser.Package{
+			{
+				CPV: "x11-terms/alacritty-0.12.3",
+			},
+		},
+	}
+
+	var buf bytes.Buffer
+	assert.NilError(t, index.EncodeInto(&buf))
+	assert.Equal(t, "ARCH: arm64\n\nCPV: x11-terms/alacritty-0.12.3\n\n", buf.String())
 }