Skip to content

fix a bug with fastly cdn for registry.k8s.io #9875

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
upodroid wants to merge 1 commit into
base: main
Choose a base branch
from
+38 −5
Open

fix a bug with fastly cdn for registry.k8s.io #9875

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion infra/fastly/terraform/cdn/fastly-log-format.tftpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{ "ddsource": "fastly", "ddtags": "app:${dd_app},env:${dd_env}", "service": "${service_name}", "date": "%%{begin:%Y-%m-%dT%H:%M:%S%Z}t", "time_start": "%%{begin:%Y-%m-%dT%H:%M:%S%Z}t", "time_end": "%%{end:%Y-%m-%dT%H:%M:%S%Z}t", "http": { "request_time_ms": %D, "method": "%m", "url": "%%{json.escape(req.url)}V", "useragent": "%%{User-Agent}i", "referer": "%%{Referer}i", "protocol": "%H", "request_x_forwarded_for": "%%{X-Forwarded-For}i", "status_code": "%s" }, "network": { "client": { "ip": "%h", "name": "%%{client.as.name}V", "number": "%%{client.as.number}V", "connection_speed": "%%{client.geo.conn_speed}V" }, "destination": { "ip": "%A" }, "geoip": { "geo_city": "%%{client.geo.city.utf8}V", "geo_country_code": "%%{client.geo.country_code}V", "geo_continent_code": "%%{client.geo.continent_code}V", "geo_region": "%%{client.geo.region}V" }, "bytes_written": %B, "bytes_read": %%{req.body_bytes_read}V }, "host": "%%{if(req.http.Fastly-Orig-Host, req.http.Fastly-Orig-Host, req.http.Host)}V", "origin_host": "%v", "is_ipv6": %%{if(req.is_ipv6, "true", "false")}V, "is_tls": %%{if(req.is_ssl, "true", "false")}V, "tls_client_protocol": "%%{json.escape(tls.client.protocol)}V", "tls_client_servername": "%%{json.escape(tls.client.servername)}V", "tls_client_cipher": "%%{json.escape(tls.client.cipher)}V", "tls_client_cipher_sha": "%%{json.escape(tls.client.ciphers_sha)}V", "tls_client_tlsexts_sha": "%%{json.escape(tls.client.tlsexts_sha)}V", "is_h2": %%{if(fastly_info.is_h2, "true", "false")}V, "is_h2_push": %%{if(fastly_info.h2.is_push, "true", "false")}V, "h2_stream_id": "%%{fastly_info.h2.stream_id}V", "request_accept_content": "%%{Accept}i", "request_accept_language": "%%{Accept-Language}i", "request_accept_encoding": "%%{Accept-Encoding}i", "request_accept_charset": "%%{Accept-Charset}i", "request_connection": "%%{Connection}i", "request_dnt": "%%{DNT}i", "request_forwarded": "%%{Forwarded}i", "request_via": "%%{Via}i", "request_cache_control": "%%{Cache-Control}i", "request_x_requested_with": "%%{X-Requested-With}i", "request_x_att_device_id": "%%{X-ATT-Device-Id}i", "content_type": "%%{Content-Type}o", "is_cacheable": %%{if(fastly_info.state~"^(HIT|MISS)$", "true","false")}V, "response_age": "%%{Age}o", "response_cache_control": "%%{Cache-Control}o", "response_expires": "%%{Expires}o", "response_last_modified": "%%{Last-Modified}o", "response_tsv": "%%{TSV}o", "server_datacenter": "%%{server.datacenter}V", "req_header_size": %%{req.header_bytes_read}V, "resp_header_size": %%{resp.header_bytes_written}V, "socket_cwnd": %%{client.socket.cwnd}V, "socket_nexthop": "%%{client.socket.nexthop}V", "socket_tcpi_rcv_mss": %%{client.socket.tcpi_rcv_mss}V, "socket_tcpi_snd_mss": %%{client.socket.tcpi_snd_mss}V, "socket_tcpi_rtt": %%{client.socket.tcpi_rtt}V, "socket_tcpi_rttvar": %%{client.socket.tcpi_rttvar}V, "socket_tcpi_rcv_rtt": %%{client.socket.tcpi_rcv_rtt}V, "socket_tcpi_rcv_space": %%{client.socket.tcpi_rcv_space}V, "socket_tcpi_last_data_sent": %%{client.socket.tcpi_last_data_sent}V, "socket_tcpi_total_retrans": %%{client.socket.tcpi_total_retrans}V, "socket_tcpi_delta_retrans": %%{client.socket.tcpi_delta_retrans}V, "socket_ploss": %%{client.socket.ploss}V }
{ "ddsource": "fastly", "ddtags": "app:${dd_app},env:${dd_env}", "service": "${service_name}", "date": "%%{begin:%Y-%m-%dT%H:%M:%S%Z}t", "time_start": "%%{begin:%Y-%m-%dT%H:%M:%S%Z}t", "time_end": "%%{end:%Y-%m-%dT%H:%M:%S%Z}t", "http": { "request_time_ms": %D, "method": "%m", "url": "%%{json.escape(req.url)}V", "request_range": "%%{json.escape(req.http.Range)}V", "request_id": "%%{json.escape(req.http.X-Request-Id-Param)}V", "useragent": "%%{User-Agent}i", "referer": "%%{Referer}i", "protocol": "%H", "request_x_forwarded_for": "%%{X-Forwarded-For}i", "status_code": "%s" }, "network": { "client": { "ip": "%h", "name": "%%{client.as.name}V", "number": "%%{client.as.number}V", "connection_speed": "%%{client.geo.conn_speed}V" }, "destination": { "ip": "%A" }, "geoip": { "geo_city": "%%{client.geo.city.utf8}V", "geo_country_code": "%%{client.geo.country_code}V", "geo_continent_code": "%%{client.geo.continent_code}V", "geo_region": "%%{client.geo.region}V" }, "bytes_written": %B, "bytes_read": %%{req.body_bytes_read}V }, "host": "%%{if(req.http.Fastly-Orig-Host, req.http.Fastly-Orig-Host, req.http.Host)}V", "origin_host": "%v", "is_ipv6": %%{if(req.is_ipv6, "true", "false")}V, "is_tls": %%{if(req.is_ssl, "true", "false")}V, "tls_client_protocol": "%%{json.escape(tls.client.protocol)}V", "tls_client_servername": "%%{json.escape(tls.client.servername)}V", "tls_client_cipher": "%%{json.escape(tls.client.cipher)}V", "tls_client_cipher_sha": "%%{json.escape(tls.client.ciphers_sha)}V", "tls_client_tlsexts_sha": "%%{json.escape(tls.client.tlsexts_sha)}V", "is_h2": %%{if(fastly_info.is_h2, "true", "false")}V, "is_h2_push": %%{if(fastly_info.h2.is_push, "true", "false")}V, "h2_stream_id": "%%{fastly_info.h2.stream_id}V", "request_accept_content": "%%{Accept}i", "request_accept_language": "%%{Accept-Language}i", "request_accept_encoding": "%%{Accept-Encoding}i", "request_accept_charset": "%%{Accept-Charset}i", "request_connection": "%%{Connection}i", "request_dnt": "%%{DNT}i", "request_forwarded": "%%{Forwarded}i", "request_via": "%%{Via}i", "request_cache_control": "%%{Cache-Control}i", "request_x_requested_with": "%%{X-Requested-With}i", "request_x_att_device_id": "%%{X-ATT-Device-Id}i", "content_type": "%%{Content-Type}o", "is_cacheable": %%{if(fastly_info.state~"^(HIT|MISS)$", "true","false")}V, "cache_state": "%%{fastly_info.state}V", "restart_count": %%{req.restarts}V, "response_age": "%%{Age}o", "response_cache_control": "%%{Cache-Control}o", "response_expires": "%%{Expires}o", "response_last_modified": "%%{Last-Modified}o", "response_tsv": "%%{TSV}o", "server_datacenter": "%%{server.datacenter}V", "req_header_size": %%{req.header_bytes_read}V, "resp_header_size": %%{resp.header_bytes_written}V, "socket_cwnd": %%{client.socket.cwnd}V, "socket_nexthop": "%%{client.socket.nexthop}V", "socket_tcpi_rcv_mss": %%{client.socket.tcpi_rcv_mss}V, "socket_tcpi_snd_mss": %%{client.socket.tcpi_snd_mss}V, "socket_tcpi_rtt": %%{client.socket.tcpi_rtt}V, "socket_tcpi_rttvar": %%{client.socket.tcpi_rttvar}V, "socket_tcpi_rcv_rtt": %%{client.socket.tcpi_rcv_rtt}V, "socket_tcpi_rcv_space": %%{client.socket.tcpi_rcv_space}V, "socket_tcpi_last_data_sent": %%{client.socket.tcpi_last_data_sent}V, "socket_tcpi_total_retrans": %%{client.socket.tcpi_total_retrans}V, "socket_tcpi_delta_retrans": %%{client.socket.tcpi_delta_retrans}V, "socket_ploss": %%{client.socket.ploss}V }
1 change: 1 addition & 0 deletions infra/fastly/terraform/cdn/services.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ resource "fastly_service_vcl" "this" {
snippet {
content = <<-EOT
set req.enable_segmented_caching = true;
set segmented_caching.block_size = 4194304;
EOT
name = "Enable segment caching for large files"
priority = 60
Expand Down
40 changes: 36 additions & 4 deletions infra/fastly/terraform/cdn/vcl/binaries.vcl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,12 @@ sub vcl_recv {
set req.http.Fastly-Purge-Requires-Auth = "1";
}

# Strip client credentials so they never reach the origins; S3 rejects
# non-AWS Authorization headers (e.g. registry Bearer tokens) with a 400.
# Private buckets are re-signed by the auth subs in vcl_miss/vcl_pass.
# cri-o/skopeo/podman pass it along and we need to strip it
unset req.http.Authorization;

# Prevent edge from caching stale content served from shield
# https://developer.fastly.com/learning/concepts/stale/#shielding-considerations
if (fastly.ff.visits_this_service != 0) {
Expand All @@ -22,9 +28,13 @@ sub vcl_recv {
if (regsub(req.url.path, {"%3[Aa]"}, ":") !~ "^/containers/images/sha256:[a-f0-9]{64}$") {
error 619;
}
# Capture rid, then strip the query string so it doesn't fragment the cache
set req.http.X-Request-Id-Param = subfield(req.url.qs, "rid", "&");
set req.url = req.url.path;
# Capture rid, then strip the query string so it doesn't fragment the cache.
# Only on the edge: the shield receives the stripped URL and would clobber
# the header forwarded from the edge with an empty value.
if (fastly.ff.visits_this_service == 0) {
set req.http.X-Request-Id-Param = subfield(req.url.qs, "rid", "&");
set req.url = req.url.path;
}
%{ else ~}
# Serve index.html for the root path
if (req.url.path == "/") {
Expand Down Expand Up @@ -86,6 +96,16 @@ sub vcl_fetch {
error beresp.status;
}

if (beresp.status >= 400 && beresp.status < 500) {
set beresp.cacheable = false;
set beresp.ttl = 0s;
set beresp.stale_if_error = 0s;
set beresp.stale_while_revalidate = 0s;
unset beresp.http.Expires;
set beresp.http.Cache-Control = "private, no-store";
return(deliver);
}

if (beresp.http.Surrogate-Control !~ "(stale-while-revalidate|stale-if-error)") {
set beresp.stale_if_error = 31536000s; # 1 year
set beresp.stale_while_revalidate = 3600s; # 1 hour
Expand Down Expand Up @@ -128,7 +148,7 @@ sub vcl_fetch {
unset beresp.http.Cache-Control;
unset beresp.http.Expires;

# Set the final headers sent to the edge PoPs and Clients
# Set the final headers sent to the edge PoPs and clients.
set beresp.ttl = 24h;
set beresp.http.Cache-Control = "public, max-age=${cache_ttl}";

Expand Down Expand Up @@ -313,3 +333,15 @@ sub vcl_pass {
}
#FASTLY pass
}

sub vcl_log {
#FASTLY log

%{~ if contains([for bucket in bucket_configs : bucket.name], "prod-registry-k8s-io-us-east-2") ~}
# Emit the rid so it's visible in log tailing.
# Only on the edge, otherwise shielded requests are logged twice.
if (fastly.ff.visits_this_service == 0 && req.http.X-Request-Id-Param ~ "^[A-Za-z0-9._-]{1,64}$") {
log {"syslog "} req.service_id {" dd-oss-k8s :: rid="} req.http.X-Request-Id-Param {" url="} req.url.path {" status="} resp.status;
}
%{ endif ~}
}