Skip to content

Commit

Permalink
tlsobs-api: parse incoming certificates with zcrypto, then run zlint
Browse files Browse the repository at this point in the history 
Issue: mozilla#352
  • Loading branch information
@adamdecaf
adamdecaf committed Jun 9, 2018
1 parent 7010d8c commit 5ea6b95
Show file tree
Hide file tree
Showing 4 changed files with 172 additions and 28 deletions.
132 changes: 120 additions & 12 deletions certificate/certificate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,15 @@ import (
"encoding/base64"
"fmt"
"log"
"math/big"
"net"
"strconv"
"strings"
"time"

certconstraints "github.com/mozilla/tls-observatory/certificate/constraints"

zcrypto "github.com/zmap/zcrypto/x509"
)

const (
Expand Down Expand Up @@ -54,6 +57,7 @@ type Certificate struct {
CiscoUmbrellaRank int64 `json:"ciscoUmbrellaRank"`
Anomalies string `json:"anomalies,omitempty"`
MozillaPolicyV2_5 MozillaPolicy `json:"mozillaPolicyV2_5"`
ZlintFailures []string `json:"zlintFailures"`
}

type MozillaPolicy struct {
Expand Down Expand Up @@ -213,21 +217,33 @@ var PublicKeyAlgorithm = [...]string{
}

func SubjectSPKISHA256(cert *x509.Certificate) string {
return subjectSPKISHA256(cert.RawSubject, cert.RawSubjectPublicKeyInfo)
}

func subjectSPKISHA256(sub, spki []byte) string {
h := sha256.New()
h.Write(cert.RawSubject)
h.Write(cert.RawSubjectPublicKeyInfo)
h.Write(sub)
h.Write(spki)
return fmt.Sprintf("%X", h.Sum(nil))
}

func SPKISHA256(cert *x509.Certificate) string {
return spkiSHA256(cert.RawSubjectPublicKeyInfo)
}

func spkiSHA256(spki []byte) string {
h := sha256.New()
h.Write(cert.RawSubjectPublicKeyInfo)
h.Write(spki)
return fmt.Sprintf("%X", h.Sum(nil))
}

func PKPSHA256Hash(cert *x509.Certificate) string {
return pkpSHA256Hash(cert.PublicKey)
}

func pkpSHA256Hash(pubKey interface{}) string {
h := sha256.New()
switch pub := cert.PublicKey.(type) {
switch pub := pubKey.(type) {
case *rsa.PublicKey, *dsa.PublicKey, *ecdsa.PublicKey:
der, _ := x509.MarshalPKIXPublicKey(pub)
h.Write(der)
Expand Down Expand Up @@ -436,12 +452,17 @@ func getMozillaPolicyV2_5(cert *x509.Certificate) MozillaPolicy {
return MozillaPolicy{IsTechnicallyConstrained: certconstraints.IsTechnicallyConstrainedMozPolicyV2_5(cert)}
}

func getPublicKeyInfo(cert *x509.Certificate) (SubjectPublicKeyInfo, error) {
pubInfo := SubjectPublicKeyInfo{
Alg: PublicKeyAlgorithm[cert.PublicKeyAlgorithm],
func getPublicKeyInfo(algo interface{}, pubKey interface{}) (SubjectPublicKeyInfo, error) {
pubInfo := SubjectPublicKeyInfo{}

switch a := algo.(type) {
case x509.PublicKeyAlgorithm:
pubInfo.Alg = PublicKeyAlgorithm[a]
case zcrypto.PublicKeyAlgorithm:
pubInfo.Alg = PublicKeyAlgorithm[a]
}

switch pub := cert.PublicKey.(type) {
switch pub := pubKey.(type) {
case *rsa.PublicKey:
pubInfo.Size = float64(pub.N.BitLen())
pubInfo.Exponent = float64(pub.E)
Expand Down Expand Up @@ -492,7 +513,11 @@ func getPublicKeyInfo(cert *x509.Certificate) (SubjectPublicKeyInfo, error) {
}

func GetHexASN1Serial(cert *x509.Certificate) (serial string, err error) {
m, err := asn1.Marshal(cert.SerialNumber)
return getHexASN1Serial(cert.SerialNumber)
}

func getHexASN1Serial(n *big.Int) (serial string, err error) {
m, err := asn1.Marshal(n)
if err != nil {
return
}
Expand Down Expand Up @@ -523,9 +548,9 @@ func CertToStored(cert *x509.Certificate, parentSignature, domain, ip string, TS

stored.SignatureAlgorithm = SignatureAlgorithm[cert.SignatureAlgorithm]

stored.Key, err = getPublicKeyInfo(cert)
stored.Key, err = getPublicKeyInfo(cert.PublicKeyAlgorithm, cert.PublicKey)
if err != nil {
log.Printf("Failed to retrieve public key information: %v. Continuing anyway.", err)
log.Printf("CertToStored: Failed to retrieve public key information: %v. Continuing anyway.", err)
}

stored.Issuer.Country = cert.Issuer.Country
Expand Down Expand Up @@ -595,7 +620,90 @@ func (cert Certificate) ToX509() (xcert *x509.Certificate, err error) {
if err != nil {
return
}
return x509.ParseCertificate(certRaw)
return x509.ParseCertificate(certRaw) // TODO(Adam): ???
}

// FromZCrypto accepts a certificate from the zcrypto library and converts it to a
// Certificate struct suitable for storage.
func FromZCrypto(z *zcrypto.Certificate) (cert *Certificate, err error) {
// initialize []string to never store them as null
cert.ParentSignature = make([]string, 0)
cert.IPs = make([]string, 0)

cert.Version = z.Version

// store zero if we run into an error
serial, _ := getHexASN1Serial(z.SerialNumber)
cert.Serial = serial

cert.SignatureAlgorithm = SignatureAlgorithm[z.SignatureAlgorithm]

cert.Key, err = getPublicKeyInfo(z.PublicKeyAlgorithm, z.PublicKey)
if err != nil {
log.Printf("FromZCrypto: Failed to retrieve public key information: %v. Continuing anyway.", err)
}

cert.Issuer.Country = z.Issuer.Country
cert.Issuer.Organisation = z.Issuer.Organization
cert.Issuer.OrgUnit = z.Issuer.OrganizationalUnit
cert.Issuer.CommonName = z.Issuer.CommonName

cert.Subject.Country = z.Subject.Country
cert.Subject.Organisation = z.Subject.Organization
cert.Subject.OrgUnit = z.Subject.OrganizationalUnit
cert.Subject.CommonName = z.Subject.CommonName

cert.Validity.NotBefore = z.NotBefore.UTC()
cert.Validity.NotAfter = z.NotAfter.UTC()

// stored.X509v3Extensions = getCertExtensions(cert) // TODO(adam)

// stored.MozillaPolicyV2_5 = getMozillaPolicyV2_5(cert) // TODO(adam)

// TODO(adam): ??? uhh ???
// //below check tries to hack around the basic constraints extension
// //not being available in versions < 3.
// //Only the IsCa variable is set, as setting X509v3BasicConstraints
// //messes up the validation procedure.
// if cert.Version < 3 {
// stored.CA = cert.IsCA
// } else {
// if cert.BasicConstraintsValid {
// stored.X509v3BasicConstraints = "Critical"
// stored.CA = cert.IsCA
// } else {
// stored.X509v3BasicConstraints = ""
// stored.CA = false
// }
// }

t := time.Now().UTC()
cert.FirstSeenTimestamp = t
cert.LastSeenTimestamp = t

// TOOD(adam): uhh??
// cert.ParentSignature = append(cert.ParentSignature, parentSignature)

// TODO(adam): uhh??
// if !cert.IsCA {
// stored.ScanTarget = domain
// stored.IPs = append(stored.IPs, ip)
// }

// cert.ValidationInfo = make(map[string]ValidationInfo)
// cert.ValidationInfo[TSName] = *valInfo

cert.Hashes.MD5 = MD5Hash(z.Raw)
cert.Hashes.SHA1 = SHA1Hash(z.Raw)
cert.Hashes.SHA256 = SHA256Hash(z.Raw)
cert.Hashes.SPKISHA256 = spkiSHA256(z.RawSubjectPublicKeyInfo)
cert.Hashes.SubjectSPKISHA256 = subjectSPKISHA256(z.RawSubject, z.RawSubjectPublicKeyInfo)
cert.Hashes.PKPSHA256 = pkpSHA256Hash(z.PublicKey)

cert.Raw = base64.StdEncoding.EncodeToString(z.Raw)
cert.CiscoUmbrellaRank = Default_Cisco_Umbrella_Rank

return cert, nil
}

//printRawCertExtensions Print raw extension info
Expand Down
7 changes: 4 additions & 3 deletions database/certificate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,10 +132,11 @@ func (db *DB) InsertCertificate(cert *certificate.Certificate) (int64, error) {
excluded_ip_addresses,
is_technically_constrained,
cisco_umbrella_rank,
mozillaPolicyV2_5
mozillaPolicyV2_5,
zlint_failures
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13,
$14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27,
$28, $29, $30, $31, $32, $33, $34, $35)
$28, $29, $30, $31, $32, $33, $34, $35, $36)
RETURNING id`,
cert.Serial,
cert.Hashes.SHA1,
Expand Down Expand Up @@ -172,6 +173,7 @@ func (db *DB) InsertCertificate(cert *certificate.Certificate) (int64, error) {
cert.X509v3Extensions.IsTechnicallyConstrained,
cert.CiscoUmbrellaRank,
mozPolicy,
cert.ZlintFailures,
).Scan(&id)
if err != nil {
return -1, err
Expand Down Expand Up @@ -340,7 +342,6 @@ func (db *DB) UpdateCertificate(cert *certificate.Certificate) error {
cert.X509v3Extensions.IsTechnicallyConstrained,
cert.CiscoUmbrellaRank,
mozPolicy,

cert.ID,
)
return err
Expand Down
3 changes: 2 additions & 1 deletion database/schema.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,8 @@ CREATE TABLE certificates(
excluded_ip_addresses varchar[] NOT NULL DEFAULT '{}',
is_technically_constrained bool NOT NULL DEFAULT false,
cisco_umbrella_rank integer NOT NULL DEFAULT 2147483647,
mozillaPolicyV2_5 jsonb NULL
mozillaPolicyV2_5 jsonb NULL,
zlint_failures jsonb NULL,
);
CREATE INDEX certificates_sha256_fingerprint_idx ON certificates(sha256_fingerprint);
CREATE INDEX certificates_subject_idx ON certificates(subject);
Expand Down
58 changes: 46 additions & 12 deletions tlsobs-api/handlers.go
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
package main

import (
"crypto/x509"
"bytes"
"crypto/sha256"
"encoding/json"
"encoding/pem"
"fmt"
Expand All @@ -14,11 +15,12 @@ import (
"strconv"
"time"

"bytes"
"crypto/sha256"

"github.com/mozilla/tls-observatory/certificate"
pg "github.com/mozilla/tls-observatory/database"

z509 "github.com/zmap/zcrypto/x509"
"github.com/zmap/zlint"
"github.com/zmap/zlint/lints"
)

var scanRefreshRate float64
Expand Down Expand Up @@ -254,15 +256,27 @@ func PostCertificateHandler(w http.ResponseWriter, r *http.Request) {
return
}

certX509, err := x509.ParseCertificate(block.Bytes)
zcert, err := z509.ParseCertificate(block.Bytes)
if err != nil {
httpError(w, r, http.StatusBadRequest,
fmt.Sprintf("Could not parse X.509 certificate: %v", err))
return
}

certHash := certificate.SHA256Hash(certX509.Raw)
id, err := db.GetCertIDBySHA256Fingerprint(certHash)
results := zlint.LintCertificate(zcert)
if results == nil {
// bad
return
}

cert, err := certificate.FromZCrypto(zcert)
if err != nil {
httpError(w, r, http.StatusInternalServerError,
fmt.Sprintf("Error converting to mozilla Certificate type: %v", err))
return
}

id, err := db.GetCertIDBySHA256Fingerprint(cert.Hashes.SHA256)
if err != nil {
httpError(w, r, http.StatusInternalServerError,
fmt.Sprintf("Failed to lookup certificate hash in database: %v", err))
Expand All @@ -275,9 +289,13 @@ func PostCertificateHandler(w http.ResponseWriter, r *http.Request) {
return
}

var valInfo certificate.ValidationInfo
cert := certificate.CertToStored(certX509, certHash, "", "", "", &valInfo)
id, err = db.InsertCertificate(&cert)
// Run zlint over certificate now that we know it's new
cert.ZlintFailures = gatherFailingLints(results)

// cert := certificate.CertToStored(certX509, certHash, "", "", "", &valInfo)
cert.ValidationInfo = make(map[string]certificate.ValidationInfo)

id, err = db.InsertCertificate(cert)
if err != nil {
httpError(w, r, http.StatusInternalServerError,
fmt.Sprintf("Failed to store certificate in database: %v", err))
Expand All @@ -292,15 +310,15 @@ func PostCertificateHandler(w http.ResponseWriter, r *http.Request) {

// to insert the trust, first build the certificate paths, then insert one trust
// entry for each known parent of the cert
paths, err := db.GetCertPaths(&cert)
paths, err := db.GetCertPaths(cert)
if err != nil {
httpError(w, r, http.StatusInternalServerError,
fmt.Sprintf("Failed to retrieve chains from database: %v", err))
return
}
for _, parent := range paths.Parents {
cert.ValidationInfo = parent.GetValidityMap()
_, err := db.InsertTrustToDB(cert, cert.ID, parent.Cert.ID)
_, err := db.InsertTrustToDB(*cert, cert.ID, parent.Cert.ID)
if err != nil {
httpError(w, r, http.StatusInternalServerError,
fmt.Sprintf("Failed to store trust in database: %v", err))
Expand All @@ -312,6 +330,22 @@ func PostCertificateHandler(w http.ResponseWriter, r *http.Request) {
return
}

// gatherFailingLints takes results from a zlint pass and collects only the
// failing error and fatal lints.
func gatherFailingLints(rs *zlint.ResultSet) []string {
if !rs.ErrorsPresent && !rs.FatalsPresent {
return nil
}

var out []string
for name, res := range rs.Results {
if res.Status == lints.Error || res.Status == lints.Fatal {
out = append(out, fmt.Sprintf("%s: %s", name, res.Details))
}
}
return out
}

// PathsHandler handles the /paths endpoint of the api.
// It queries the database for the provided cert ids or sha256 and returns
// its chain of trust in JSON.
Expand Down

0 comments on commit 5ea6b95

Please sign in to comment.