diff --git a/certificate/certificate.go b/certificate/certificate.go index 10c0df8f1..4b24fb171 100644 --- a/certificate/certificate.go +++ b/certificate/certificate.go @@ -12,12 +12,15 @@ import ( "encoding/base64" "fmt" "log" + "math/big" "net" "strconv" "strings" "time" certconstraints "github.com/mozilla/tls-observatory/certificate/constraints" + + zcrypto "github.com/zmap/zcrypto/x509" ) const ( @@ -54,6 +57,7 @@ type Certificate struct { CiscoUmbrellaRank int64 `json:"ciscoUmbrellaRank"` Anomalies string `json:"anomalies,omitempty"` MozillaPolicyV2_5 MozillaPolicy `json:"mozillaPolicyV2_5"` + ZlintFailures []string `json:"zlintFailures"` } type MozillaPolicy struct { @@ -213,21 +217,33 @@ var PublicKeyAlgorithm = [...]string{ } func SubjectSPKISHA256(cert *x509.Certificate) string { + return subjectSPKISHA256(cert.RawSubject, cert.RawSubjectPublicKeyInfo) +} + +func subjectSPKISHA256(sub, spki []byte) string { h := sha256.New() - h.Write(cert.RawSubject) - h.Write(cert.RawSubjectPublicKeyInfo) + h.Write(sub) + h.Write(spki) return fmt.Sprintf("%X", h.Sum(nil)) } func SPKISHA256(cert *x509.Certificate) string { + return spkiSHA256(cert.RawSubjectPublicKeyInfo) +} + +func spkiSHA256(spki []byte) string { h := sha256.New() - h.Write(cert.RawSubjectPublicKeyInfo) + h.Write(spki) return fmt.Sprintf("%X", h.Sum(nil)) } func PKPSHA256Hash(cert *x509.Certificate) string { + return pkpSHA256Hash(cert.PublicKey) +} + +func pkpSHA256Hash(pubKey interface{}) string { h := sha256.New() - switch pub := cert.PublicKey.(type) { + switch pub := pubKey.(type) { case *rsa.PublicKey, *dsa.PublicKey, *ecdsa.PublicKey: der, _ := x509.MarshalPKIXPublicKey(pub) h.Write(der) @@ -436,12 +452,17 @@ func getMozillaPolicyV2_5(cert *x509.Certificate) MozillaPolicy { return MozillaPolicy{IsTechnicallyConstrained: certconstraints.IsTechnicallyConstrainedMozPolicyV2_5(cert)} } -func getPublicKeyInfo(cert *x509.Certificate) (SubjectPublicKeyInfo, error) { - pubInfo := SubjectPublicKeyInfo{ - Alg: PublicKeyAlgorithm[cert.PublicKeyAlgorithm], +func getPublicKeyInfo(algo interface{}, pubKey interface{}) (SubjectPublicKeyInfo, error) { + pubInfo := SubjectPublicKeyInfo{} + + switch a := algo.(type) { + case x509.PublicKeyAlgorithm: + pubInfo.Alg = PublicKeyAlgorithm[a] + case zcrypto.PublicKeyAlgorithm: + pubInfo.Alg = PublicKeyAlgorithm[a] } - switch pub := cert.PublicKey.(type) { + switch pub := pubKey.(type) { case *rsa.PublicKey: pubInfo.Size = float64(pub.N.BitLen()) pubInfo.Exponent = float64(pub.E) @@ -492,7 +513,11 @@ func getPublicKeyInfo(cert *x509.Certificate) (SubjectPublicKeyInfo, error) { } func GetHexASN1Serial(cert *x509.Certificate) (serial string, err error) { - m, err := asn1.Marshal(cert.SerialNumber) + return getHexASN1Serial(cert.SerialNumber) +} + +func getHexASN1Serial(n *big.Int) (serial string, err error) { + m, err := asn1.Marshal(n) if err != nil { return } @@ -523,9 +548,9 @@ func CertToStored(cert *x509.Certificate, parentSignature, domain, ip string, TS stored.SignatureAlgorithm = SignatureAlgorithm[cert.SignatureAlgorithm] - stored.Key, err = getPublicKeyInfo(cert) + stored.Key, err = getPublicKeyInfo(cert.PublicKeyAlgorithm, cert.PublicKey) if err != nil { - log.Printf("Failed to retrieve public key information: %v. Continuing anyway.", err) + log.Printf("CertToStored: Failed to retrieve public key information: %v. Continuing anyway.", err) } stored.Issuer.Country = cert.Issuer.Country @@ -595,7 +620,90 @@ func (cert Certificate) ToX509() (xcert *x509.Certificate, err error) { if err != nil { return } - return x509.ParseCertificate(certRaw) + return x509.ParseCertificate(certRaw) // TODO(Adam): ??? +} + +// FromZCrypto accepts a certificate from the zcrypto library and converts it to a +// Certificate struct suitable for storage. +func FromZCrypto(z *zcrypto.Certificate) (cert *Certificate, err error) { + // initialize []string to never store them as null + cert.ParentSignature = make([]string, 0) + cert.IPs = make([]string, 0) + + cert.Version = z.Version + + // store zero if we run into an error + serial, _ := getHexASN1Serial(z.SerialNumber) + cert.Serial = serial + + cert.SignatureAlgorithm = SignatureAlgorithm[z.SignatureAlgorithm] + + cert.Key, err = getPublicKeyInfo(z.PublicKeyAlgorithm, z.PublicKey) + if err != nil { + log.Printf("FromZCrypto: Failed to retrieve public key information: %v. Continuing anyway.", err) + } + + cert.Issuer.Country = z.Issuer.Country + cert.Issuer.Organisation = z.Issuer.Organization + cert.Issuer.OrgUnit = z.Issuer.OrganizationalUnit + cert.Issuer.CommonName = z.Issuer.CommonName + + cert.Subject.Country = z.Subject.Country + cert.Subject.Organisation = z.Subject.Organization + cert.Subject.OrgUnit = z.Subject.OrganizationalUnit + cert.Subject.CommonName = z.Subject.CommonName + + cert.Validity.NotBefore = z.NotBefore.UTC() + cert.Validity.NotAfter = z.NotAfter.UTC() + + // stored.X509v3Extensions = getCertExtensions(cert) // TODO(adam) + + // stored.MozillaPolicyV2_5 = getMozillaPolicyV2_5(cert) // TODO(adam) + + // TODO(adam): ??? uhh ??? + // //below check tries to hack around the basic constraints extension + // //not being available in versions < 3. + // //Only the IsCa variable is set, as setting X509v3BasicConstraints + // //messes up the validation procedure. + // if cert.Version < 3 { + // stored.CA = cert.IsCA + // } else { + // if cert.BasicConstraintsValid { + // stored.X509v3BasicConstraints = "Critical" + // stored.CA = cert.IsCA + // } else { + // stored.X509v3BasicConstraints = "" + // stored.CA = false + // } + // } + + t := time.Now().UTC() + cert.FirstSeenTimestamp = t + cert.LastSeenTimestamp = t + + // TOOD(adam): uhh?? + // cert.ParentSignature = append(cert.ParentSignature, parentSignature) + + // TODO(adam): uhh?? + // if !cert.IsCA { + // stored.ScanTarget = domain + // stored.IPs = append(stored.IPs, ip) + // } + + // cert.ValidationInfo = make(map[string]ValidationInfo) + // cert.ValidationInfo[TSName] = *valInfo + + cert.Hashes.MD5 = MD5Hash(z.Raw) + cert.Hashes.SHA1 = SHA1Hash(z.Raw) + cert.Hashes.SHA256 = SHA256Hash(z.Raw) + cert.Hashes.SPKISHA256 = spkiSHA256(z.RawSubjectPublicKeyInfo) + cert.Hashes.SubjectSPKISHA256 = subjectSPKISHA256(z.RawSubject, z.RawSubjectPublicKeyInfo) + cert.Hashes.PKPSHA256 = pkpSHA256Hash(z.PublicKey) + + cert.Raw = base64.StdEncoding.EncodeToString(z.Raw) + cert.CiscoUmbrellaRank = Default_Cisco_Umbrella_Rank + + return cert, nil } //printRawCertExtensions Print raw extension info diff --git a/database/certificate.go b/database/certificate.go index d3c078626..a6f590317 100644 --- a/database/certificate.go +++ b/database/certificate.go @@ -132,10 +132,11 @@ func (db *DB) InsertCertificate(cert *certificate.Certificate) (int64, error) { excluded_ip_addresses, is_technically_constrained, cisco_umbrella_rank, - mozillaPolicyV2_5 + mozillaPolicyV2_5, + zlint_failures ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, - $28, $29, $30, $31, $32, $33, $34, $35) + $28, $29, $30, $31, $32, $33, $34, $35, $36) RETURNING id`, cert.Serial, cert.Hashes.SHA1, @@ -172,6 +173,7 @@ func (db *DB) InsertCertificate(cert *certificate.Certificate) (int64, error) { cert.X509v3Extensions.IsTechnicallyConstrained, cert.CiscoUmbrellaRank, mozPolicy, + cert.ZlintFailures, ).Scan(&id) if err != nil { return -1, err @@ -340,7 +342,6 @@ func (db *DB) UpdateCertificate(cert *certificate.Certificate) error { cert.X509v3Extensions.IsTechnicallyConstrained, cert.CiscoUmbrellaRank, mozPolicy, - cert.ID, ) return err diff --git a/database/schema.sql b/database/schema.sql index 97fd867d9..1ba638c71 100644 --- a/database/schema.sql +++ b/database/schema.sql @@ -45,7 +45,8 @@ CREATE TABLE certificates( excluded_ip_addresses varchar[] NOT NULL DEFAULT '{}', is_technically_constrained bool NOT NULL DEFAULT false, cisco_umbrella_rank integer NOT NULL DEFAULT 2147483647, - mozillaPolicyV2_5 jsonb NULL + mozillaPolicyV2_5 jsonb NULL, + zlint_failures jsonb NULL, ); CREATE INDEX certificates_sha256_fingerprint_idx ON certificates(sha256_fingerprint); CREATE INDEX certificates_subject_idx ON certificates(subject); diff --git a/tlsobs-api/handlers.go b/tlsobs-api/handlers.go index 8aee3c613..400391907 100644 --- a/tlsobs-api/handlers.go +++ b/tlsobs-api/handlers.go @@ -1,7 +1,8 @@ package main import ( - "crypto/x509" + "bytes" + "crypto/sha256" "encoding/json" "encoding/pem" "fmt" @@ -14,11 +15,12 @@ import ( "strconv" "time" - "bytes" - "crypto/sha256" - "github.com/mozilla/tls-observatory/certificate" pg "github.com/mozilla/tls-observatory/database" + + z509 "github.com/zmap/zcrypto/x509" + "github.com/zmap/zlint" + "github.com/zmap/zlint/lints" ) var scanRefreshRate float64 @@ -254,15 +256,27 @@ func PostCertificateHandler(w http.ResponseWriter, r *http.Request) { return } - certX509, err := x509.ParseCertificate(block.Bytes) + zcert, err := z509.ParseCertificate(block.Bytes) if err != nil { httpError(w, r, http.StatusBadRequest, fmt.Sprintf("Could not parse X.509 certificate: %v", err)) return } - certHash := certificate.SHA256Hash(certX509.Raw) - id, err := db.GetCertIDBySHA256Fingerprint(certHash) + results := zlint.LintCertificate(zcert) + if results == nil { + // bad + return + } + + cert, err := certificate.FromZCrypto(zcert) + if err != nil { + httpError(w, r, http.StatusInternalServerError, + fmt.Sprintf("Error converting to mozilla Certificate type: %v", err)) + return + } + + id, err := db.GetCertIDBySHA256Fingerprint(cert.Hashes.SHA256) if err != nil { httpError(w, r, http.StatusInternalServerError, fmt.Sprintf("Failed to lookup certificate hash in database: %v", err)) @@ -275,9 +289,13 @@ func PostCertificateHandler(w http.ResponseWriter, r *http.Request) { return } - var valInfo certificate.ValidationInfo - cert := certificate.CertToStored(certX509, certHash, "", "", "", &valInfo) - id, err = db.InsertCertificate(&cert) + // Run zlint over certificate now that we know it's new + cert.ZlintFailures = gatherFailingLints(results) + + // cert := certificate.CertToStored(certX509, certHash, "", "", "", &valInfo) + cert.ValidationInfo = make(map[string]certificate.ValidationInfo) + + id, err = db.InsertCertificate(cert) if err != nil { httpError(w, r, http.StatusInternalServerError, fmt.Sprintf("Failed to store certificate in database: %v", err)) @@ -292,7 +310,7 @@ func PostCertificateHandler(w http.ResponseWriter, r *http.Request) { // to insert the trust, first build the certificate paths, then insert one trust // entry for each known parent of the cert - paths, err := db.GetCertPaths(&cert) + paths, err := db.GetCertPaths(cert) if err != nil { httpError(w, r, http.StatusInternalServerError, fmt.Sprintf("Failed to retrieve chains from database: %v", err)) @@ -300,7 +318,7 @@ func PostCertificateHandler(w http.ResponseWriter, r *http.Request) { } for _, parent := range paths.Parents { cert.ValidationInfo = parent.GetValidityMap() - _, err := db.InsertTrustToDB(cert, cert.ID, parent.Cert.ID) + _, err := db.InsertTrustToDB(*cert, cert.ID, parent.Cert.ID) if err != nil { httpError(w, r, http.StatusInternalServerError, fmt.Sprintf("Failed to store trust in database: %v", err)) @@ -312,6 +330,22 @@ func PostCertificateHandler(w http.ResponseWriter, r *http.Request) { return } +// gatherFailingLints takes results from a zlint pass and collects only the +// failing error and fatal lints. +func gatherFailingLints(rs *zlint.ResultSet) []string { + if !rs.ErrorsPresent && !rs.FatalsPresent { + return nil + } + + var out []string + for name, res := range rs.Results { + if res.Status == lints.Error || res.Status == lints.Fatal { + out = append(out, fmt.Sprintf("%s: %s", name, res.Details)) + } + } + return out +} + // PathsHandler handles the /paths endpoint of the api. // It queries the database for the provided cert ids or sha256 and returns // its chain of trust in JSON.