Limitissuer

cmd/tesseract/README.md

@@ -54,7 +54,7 @@ means no upper bound on the accepted range. RFC3339 UTC format, e.g:
 signing algorithms. This flag is a temporary solution to allow chains submitted
 by Chrome's Merge Delay Monitor Root. It will eventually be removed and chains
 using such algorithms will be rejected.
-- `limit_old_submissions`: This optional flag can be set define a limit on how
+- `rate_limit_old_not_before`: This optional flag can be set define a limit on how
 many "old" certificates and precertificates will be accepted per second.
 The flag value should be of the form `<age>:<limit>`, where `<limit>` is a
 per-second rate limit, and `<age>` defines how old a given submission's
@@ -119,7 +119,7 @@ This flag is on by default.
 
 #### Antispam
 
-The `pushback_max_antispam_lag`, `pushback_max_dedup_in_flight` and
+The `pushback_max_antispam_lag`, `rate_limit_dedup` and
 `inmemory_antispam_cache_size` flags control how [TesseraCT's Antispam feature](/docs/architecture.md#antispam)
 works, which itself is built on top of [Tessera's Antispam](https://github.com/transparency-dev/tessera?tab=readme-ov-file#antispam)
 capabilities. It is composed of three main steps:
@@ -146,14 +146,13 @@ calls faster, and provides optimistic coverage for entries submitted _very_
 recently and which have not yet been processed by the asynchronous process in
 `(1)`.
 
-The `pushback_max_dedup_in_flight` flag rate limits how many concurrent `add-*`
-requests identified as duplicates will be processed by the
-**synchronous** process in `(3)` wich fetches entries and extracts information
-required to build SCTs. When this value is exceeded, TesseraCT returns
-`429 -Too Many Requests` to subsequent **duplicate** `add-*` requests only.
-Non-duplicate `add-*` requests are not impacted, and can still be processed.
-This limits the amount of resources TesseraCT spends on servicing duplicate
-requests.
+The `rate_limit_dedup` flag rate limits how many concurrent `add-*` requests
+identified as duplicates will be processed by the **synchronous** process in
+`(3)` which fetches entries and extracts information required to build SCTs.
+When this value is exceeded, TesseraCT returns `429 -Too Many Requests` to
+subsequent **duplicate** `add-*` requests only. Non-duplicate `add-*` requests
+are not impacted, and can still be processed. This limits the amount of
+resources TesseraCT spends on servicing duplicate requests.
 
 ### Setup
 

cmd/tesseract/aws/main.go

@@ -47,16 +47,16 @@ import (
 func init() {
 	flag.Var(&notAfterStart, "not_after_start", "Start of the range of acceptable NotAfter values, inclusive. Leaving this unset or empty implies no lower bound to the range. RFC3339 UTC format, e.g: 2024-01-02T15:04:05Z.")
 	flag.Var(&notAfterLimit, "not_after_limit", "Cut off point of notAfter dates - only notAfter dates strictly *before* notAfterLimit will be accepted. Leaving this unset or empty means no upper bound on the accepted range. RFC3339 UTC format, e.g: 2024-01-02T15:04:05Z.")
-	flag.UintVar(&pushbackMaxDedupInFlight, "pushback_max_dedup_in_flight", 100, "Maximum number of number of in-flight duplicate add requests - i.e. the number of requests matching entries that have already been integrated, but need to be fetched by the client to retrieve their timestamp. When 0, duplicate entries are always pushed back.")
+	flag.Float64Var(&dedupRL, "rate_limit_dedup", 100, "Rate limit for resolving duplicate submissions, in requests per second - i.e. duplicate requests for already integrated entries, which need to be fetched from the log storage by TesseraCT to extract their timestamp. When 0, all duplicate submissions are rejected. When negative, no rate limit is applied.")
 	// DEPRECATED: will be removed shortly
-	flag.UintVar(&pushbackMaxDedupInFlight, "pushback_max_dedupe_in_flight", 100, "DEPRECATED: use pushback_max_dedup_in_flight. Maximum number of number of in-flight duplicate add requests - i.e. the number of requests matching entries that have already been integrated, but need to be fetched by the client to retrieve their timestamp. When 0, duplicate entries are always pushed back.")
+	flag.Float64Var(&dedupRL, "pushback_max_dedupe_in_flight", 100, "DEPRECATED: use rate_limit_dedup. Maximum number of number of in-flight duplicate add requests - i.e. the number of requests matching entries that have already been integrated, but need to be fetched by the client to retrieve their timestamp. When 0, duplicate entries are always pushed back.")
 }
 
 // Global flags that affect all log instances.
 var (
-	notAfterStart            timestampFlag
-	notAfterLimit            timestampFlag
-	pushbackMaxDedupInFlight uint
+	notAfterStart timestampFlag
+	notAfterLimit timestampFlag
+	dedupRL       float64
 
 	// Functionality flags
 	httpEndpoint             = flag.String("http_endpoint", "localhost:6962", "Endpoint for HTTP (host:port).")
@@ -70,7 +70,8 @@ var (
 	rejectExtensions         = flag.String("reject_extension", "", "A list of X.509 extension OIDs, in dotted string form (e.g. '2.3.4.5') which, if present, should cause submissions to be rejected.")
 	acceptSHA1               = flag.Bool("accept_sha1_signing_algorithms", true, "If true, accept chains that use SHA-1 based signing algorithms. This flag will eventually be removed, and such algorithms will be rejected.")
 	enablePublicationAwaiter = flag.Bool("enable_publication_awaiter", true, "If true then the certificate is integrated into log before returning the response.")
-	limitOldCerts            = flag.String("limit_old_submissions", "", "Optionally rate limits submissions with old notBefore dates. Expects a value of with the format: \"<go duration>:<rate limit>\", e.g. \"30d:50\" would impose a limit of 50 certs/s on submissions whose notBefore date is >= 30days old.")
+	notBeforeRL              = flag.String("rate_limit_old_not_before", "", "Optionally rate limits submissions with old notBefore dates. Expects a value of with the format: \"<go duration>:<rate limit>\", e.g. \"30d:50\" would impose a limit of 50 certs/s on submissions whose notBefore date is >= 30days old.")
+	issuerRL                 = flag.Float64("rate_limit_per_issuer", -1, "Optionally rate limits submissions per issuer per second. Disabled when null or negative.")
 
 	// Performance flags
 	httpDeadline              = flag.Duration("http_deadline", time.Second*10, "Deadline for HTTP requests.")
@@ -140,7 +141,9 @@ eventually go away. See /internal/lax509/README.md for more information.`)
 	}
 
 	hOpts := tesseract.LogHandlerOpts{
-		OldSubmissionLimit: rateLimitFromFlags(),
+		NotBeforeRL: notBeforeRLFromFlags(),
+		IssuerRL:    *issuerRL,
+		DedupRL:     dedupRL,
 	}
 	logHandler, err := tesseract.NewLogHandler(ctx, *origin, signer, chainValidationConfig, newAWSStorage, *httpDeadline, *maskInternalErrors, *pathPrefix, hOpts)
 	if err != nil {
@@ -242,11 +245,10 @@ func newAWSStorage(ctx context.Context, signer note.Signer) (*storage.CTStorage,
 	}
 
 	sopts := storage.CTStorageOptions{
-		Appender:         appender,
-		Reader:           reader,
-		IssuerStorage:    issuerStorage,
-		EnableAwaiter:    *enablePublicationAwaiter,
-		MaxDedupInFlight: pushbackMaxDedupInFlight,
+		Appender:      appender,
+		Reader:        reader,
+		IssuerStorage: issuerStorage,
+		EnableAwaiter: *enablePublicationAwaiter,
 	}
 
 	return storage.NewCTStorage(ctx, &sopts)
@@ -362,21 +364,21 @@ func antispamMySQLConfig() *mysql.Config {
 	}
 }
 
-func rateLimitFromFlags() *tesseract.OldSubmissionLimit {
-	if *limitOldCerts == "" {
+func notBeforeRLFromFlags() *tesseract.NotBeforeRL {
+	if *notBeforeRL == "" {
 		return nil
 	}
-	bits := strings.Split(*limitOldCerts, ":")
+	bits := strings.Split(*notBeforeRL, ":")
 	if len(bits) != 2 {
-		klog.Exitf("Invalid format for --limit_old_submissions flag")
+		klog.Exitf("Invalid format for --rate_limit_old_not_before flag")
 	}
 	a, err := time.ParseDuration(bits[0])
 	if err != nil {
-		klog.Exitf("Invalid age passed to --limit_old_submissions flag %q: %v", bits[0], err)
+		klog.Exitf("Invalid age passed to --rate_limit_old_not_before flag %q: %v", bits[0], err)
 	}
 	l, err := strconv.ParseFloat(bits[1], 64)
 	if err != nil {
-		klog.Exitf("Invalid rate limit passed to --limit_old_submissions %q: %v", bits[1], err)
+		klog.Exitf("Invalid rate limit passed to --rate_limit_old_not_before %q: %v", bits[1], err)
 	}
-	return &tesseract.OldSubmissionLimit{AgeThreshold: a, RateLimit: l}
+	return &tesseract.NotBeforeRL{AgeThreshold: a, RateLimit: l}
 }

cmd/tesseract/gcp/main.go

@@ -46,17 +46,17 @@ func init() {
 	flag.Var(&notAfterStart, "not_after_start", "Start of the range of acceptable NotAfter values, inclusive. Leaving this unset or empty implies no lower bound to the range. RFC3339 UTC format, e.g: 2024-01-02T15:04:05Z.")
 	flag.Var(&notAfterLimit, "not_after_limit", "Cut off point of notAfter dates - only notAfter dates strictly *before* notAfterLimit will be accepted. Leaving this unset or empty means no upper bound on the accepted range. RFC3339 UTC format, e.g: 2024-01-02T15:04:05Z.")
 	flag.Var(&additionalSigners, "additional_signer_private_key_secret_name", "Private key secret name for additional Ed25519 checkpoint signatures, may be supplied multiple times. Format: projects/{projectId}/secrets/{secretName}/versions/{secretVersion}.")
-	flag.UintVar(&pushbackMaxDedupInFlight, "pushback_max_dedup_in_flight", 100, "Maximum number of number of in-flight duplicate add requests - i.e. the number of requests matching entries that have already been integrated, but need to be fetched by the client to retrieve their timestamp. When 0, duplicate entries are always pushed back.")
+	flag.Float64Var(&dedupRL, "rate_limit_dedup", 100, "Rate limit for resolving duplicate submissions, in requests per second - i.e. duplicate requests for already integrated entries, which need to be fetched from the log storage by TesseraCT to extract their timestamp. When 0, all duplicate submissions are rejected. When negative, no rate limit is applied.")
 	// DEPRECATED: will be removed shortly
-	flag.UintVar(&pushbackMaxDedupInFlight, "pushback_max_dedupe_in_flight", 100, "DEPRECATED: use pushback_max_dedup_in_flight. Maximum number of number of in-flight duplicate add requests - i.e. the number of requests matching entries that have already been integrated, but need to be fetched by the client to retrieve their timestamp. When 0, duplicate entries are always pushed back.")
+	flag.Float64Var(&dedupRL, "pushback_max_dedupe_in_flight", 100, "DEPRECATED: use rate_limit_dedup. Maximum number of number of in-flight duplicate add requests - i.e. the number of requests matching entries that have already been integrated, but need to be fetched by the client to retrieve their timestamp. When 0, duplicate entries are always pushed back.")
 }
 
 // Global flags that affect all log instances.
 var (
-	notAfterStart            timestampFlag
-	notAfterLimit            timestampFlag
-	additionalSigners        multiStringFlag
-	pushbackMaxDedupInFlight uint
+	notAfterStart     timestampFlag
+	notAfterLimit     timestampFlag
+	additionalSigners multiStringFlag
+	dedupRL           float64
 
 	// Functionality flags
 	httpEndpoint             = flag.String("http_endpoint", "localhost:6962", "Endpoint for HTTP (host:port).")
@@ -71,7 +71,8 @@ var (
 	acceptSHA1               = flag.Bool("accept_sha1_signing_algorithms", true, "If true, accept chains that use SHA-1 based signing algorithms. This flag will eventually be removed, and such algorithms will be rejected.")
 	enablePublicationAwaiter = flag.Bool("enable_publication_awaiter", true, "If true then the certificate is integrated into log before returning the response.")
 	witnessPolicyFile        = flag.String("witness_policy_file", "", "(Optional) Path to the file containing the witness policy in the format described at https://git.glasklar.is/sigsum/core/sigsum-go/-/blob/main/doc/policy.md")
-	limitOldCerts            = flag.String("limit_old_submissions", "", "Optionally rate limits submissions with old notBefore dates. Expects a value of with the format: \"<go duration>:<rate limit>\", e.g. \"30d:50\" would impose a limit of 50 certs/s on submissions whose notBefore date is >= 30days old.")
+	notBeforeRL              = flag.String("rate_limit_old_not_before", "", "Optionally rate limits submissions with old notBefore dates. Expects a value of with the format: \"<go duration>:<rate limit>\", e.g. \"30d:50\" would impose a limit of 50 certs/s on submissions whose notBefore date is >= 30days old.")
+	issuerRL                 = flag.Float64("rate_limit_per_issuer", -1, "Optionally rate limits submissions per issuer per second. Disabled when null or negative.")
 
 	// Performance flags
 	httpDeadline              = flag.Duration("http_deadline", time.Second*10, "Deadline for HTTP requests.")
@@ -127,7 +128,9 @@ eventually go away. See /internal/lax509/README.md for more information.`)
 	}
 
 	hOpts := tesseract.LogHandlerOpts{
-		OldSubmissionLimit: rateLimitFromFlags(),
+		NotBeforeRL: notBeforeRLFromFlags(),
+		IssuerRL:    *issuerRL,
+		DedupRL:     dedupRL,
 	}
 	logHandler, err := tesseract.NewLogHandler(ctx, *origin, signer, chainValidationConfig, newGCPStorage, *httpDeadline, *maskInternalErrors, *pathPrefix, hOpts)
 	if err != nil {
@@ -279,11 +282,10 @@ func newGCPStorage(ctx context.Context, signer note.Signer) (*storage.CTStorage,
 	}
 
 	sopts := storage.CTStorageOptions{
-		Appender:         appender,
-		Reader:           reader,
-		IssuerStorage:    issuerStorage,
-		EnableAwaiter:    *enablePublicationAwaiter,
-		MaxDedupInFlight: pushbackMaxDedupInFlight,
+		Appender:      appender,
+		Reader:        reader,
+		IssuerStorage: issuerStorage,
+		EnableAwaiter: *enablePublicationAwaiter,
 	}
 
 	return storage.NewCTStorage(ctx, &sopts)
@@ -314,21 +316,21 @@ func (t *timestampFlag) Set(w string) error {
 	return nil
 }
 
-func rateLimitFromFlags() *tesseract.OldSubmissionLimit {
-	if *limitOldCerts == "" {
+func notBeforeRLFromFlags() *tesseract.NotBeforeRL {
+	if *notBeforeRL == "" {
 		return nil
 	}
-	bits := strings.Split(*limitOldCerts, ":")
+	bits := strings.Split(*notBeforeRL, ":")
 	if len(bits) != 2 {
-		klog.Exitf("Invalid format for --limit_old_submissions flag")
+		klog.Exitf("Invalid format for --rate_limit_old_not_before flag")
 	}
 	a, err := time.ParseDuration(bits[0])
 	if err != nil {
-		klog.Exitf("Invalid age passed to --limit_old_submissions flag %q: %v", bits[0], err)
+		klog.Exitf("Invalid age passed to --rate_limit_old_not_before flag %q: %v", bits[0], err)
 	}
 	l, err := strconv.ParseFloat(bits[1], 64)
 	if err != nil {
-		klog.Exitf("Invalid rate limit passed to --limit_old_submissions %q: %v", bits[1], err)
+		klog.Exitf("Invalid rate limit passed to --rate_limit_old_not_before %q: %v", bits[1], err)
 	}
-	return &tesseract.OldSubmissionLimit{AgeThreshold: a, RateLimit: l}
+	return &tesseract.NotBeforeRL{AgeThreshold: a, RateLimit: l}
 }