chore(stats): retire legacy s3 lake (#52515)

This commit is contained in:
Adam
2026-10-01 22:21:08 -05:00
committed by GitHub
Unverified
parent a79ecfe109
commit 1ddb0873ae
2 changed files with 4 additions and 176 deletions
-89
View File
@@ -1,89 +0,0 @@
// Retired in favor of the R2 lake. Nothing writes to it anymore, but the table,
// catalog and Athena workgroup stay while the anomalyco/anomaly inference and
// analytics dashboards still query its history. Pulumi reads forceDestroy and
// retainOnDelete from state at delete time, so keep those settings until the
// lake is removed.
const current = aws.getCallerIdentityOutput({})
const partition = aws.getPartitionOutput({})
const region = aws.getRegionOutput({})
const tableBucketName = `opencode-${$app.stage}-lake`
const glueCatalogName = "s3tablescatalog"
const s3TablesBucketWildcardArn = $interpolate`arn:${partition.partition}:s3tables:${region.region}:${current.accountId}:bucket/*`
export const tableBucket = new aws.s3tables.TableBucket("LakeTableBucket", {
name: tableBucketName,
forceDestroy: true,
})
new aws.cloudcontrol.Resource(
"LakeS3TablesCatalog",
{
typeName: "AWS::Glue::Catalog",
desiredState: $jsonStringify({
Name: glueCatalogName,
Description: "Federated catalog for S3 Tables",
FederatedCatalog: {
Identifier: s3TablesBucketWildcardArn,
ConnectionName: "aws:s3tables",
},
CreateDatabaseDefaultPermissions: [
{
Principal: {
DataLakePrincipalIdentifier: "IAM_ALLOWED_PRINCIPALS",
},
Permissions: ["ALL"],
},
],
CreateTableDefaultPermissions: [
{
Principal: {
DataLakePrincipalIdentifier: "IAM_ALLOWED_PRINCIPALS",
},
Permissions: ["ALL"],
},
],
AllowFullTableExternalDataAccess: "True",
}),
},
{ dependsOn: [tableBucket] },
)
const athenaResultsBucket = new aws.s3.Bucket(
"LakeAthenaResults",
{
bucket: `opencode-${$app.stage}-lake-athena-results`,
forceDestroy: true,
},
{ retainOnDelete: false },
)
// Keep the archived delivery failures until their records have been reconciled.
new aws.s3.Bucket(
"LakeFirehoseErrors",
{
bucket: `opencode-${$app.stage}-lake-firehose-errors`,
forceDestroy: true,
},
{ retainOnDelete: false },
)
new aws.athena.Workgroup("LakeAthenaWorkgroup", {
name: `opencode-${$app.stage}-lake-workgroup`,
forceDestroy: true,
configuration: {
enforceWorkgroupConfiguration: true,
publishCloudwatchMetricsEnabled: true,
// Athena bills $5/TB scanned; kill any query that would scan more than 2 TB
// so a regression cannot silently burn money. Stats sync full passes scan
// ~250 GB as of 2026-07.
bytesScannedCutoffPerQuery: 2 * 1024 ** 4,
resultConfiguration: {
outputLocation: $interpolate`s3://${athenaResultsBucket.bucket}/`,
},
},
})
export const lakeVpc = new sst.aws.Vpc("LakeVpc")
export const lakeCluster = new sst.aws.Cluster("LakeCluster", { vpc: lakeVpc })
+4 -87
View File
@@ -1,93 +1,10 @@
import { lakeCluster, tableBucket } from "./lake"
import { EMAILOCTOPUS_API_KEY } from "./app"
import { domain } from "./stage"
////////////////
// LAKE
////////////////
const inferenceNamespace = new aws.s3tables.Namespace("LakeInferenceNamespace", {
namespace: "inference",
tableBucketArn: tableBucket.arn,
})
new aws.s3tables.Table(
"LakeInferenceEventTable",
{
name: "event",
namespace: inferenceNamespace.namespace,
tableBucketArn: inferenceNamespace.tableBucketArn,
format: "ICEBERG",
metadata: {
iceberg: {
schema: {
fields: [
{ name: "event_timestamp", type: "string", required: false },
{ name: "event_date", type: "string", required: false },
{ name: "event_type", type: "string", required: false },
{ name: "dataset", type: "string", required: false },
{ name: "cf_continent", type: "string", required: false },
{ name: "cf_country", type: "string", required: false },
{ name: "cf_city", type: "string", required: false },
{ name: "cf_region", type: "string", required: false },
{ name: "cf_latitude", type: "double", required: false },
{ name: "cf_longitude", type: "double", required: false },
{ name: "cf_timezone", type: "string", required: false },
{ name: "duration", type: "double", required: false },
{ name: "request_length", type: "long", required: false },
{ name: "status", type: "int", required: false },
{ name: "ip", type: "string", required: false },
{ name: "is_stream", type: "boolean", required: false },
{ name: "session", type: "string", required: false },
{ name: "request", type: "string", required: false },
{ name: "client", type: "string", required: false },
{ name: "user_agent", type: "string", required: false },
{ name: "model", type: "string", required: false },
{ name: "model_tier", type: "string", required: false },
{ name: "model_variant", type: "string", required: false },
{ name: "source", type: "string", required: false },
{ name: "provider", type: "string", required: false },
{ name: "provider_model", type: "string", required: false },
{ name: "llm_error_code", type: "int", required: false },
{ name: "llm_error_message", type: "string", required: false },
{ name: "error_response", type: "string", required: false },
{ name: "error_type", type: "string", required: false },
{ name: "error_message", type: "string", required: false },
{ name: "error_cause", type: "string", required: false },
{ name: "error_cause2", type: "string", required: false },
{ name: "api_key", type: "string", required: false },
{ name: "workspace", type: "string", required: false },
{ name: "user_id", type: "string", required: false },
{ name: "is_subscription", type: "boolean", required: false },
{ name: "subscription", type: "string", required: false },
{ name: "response_length", type: "long", required: false },
{ name: "time_to_first_byte", type: "long", required: false },
{ name: "timestamp_first_byte", type: "long", required: false },
{ name: "timestamp_last_byte", type: "long", required: false },
{ name: "tokens_input", type: "long", required: false },
{ name: "tokens_output", type: "long", required: false },
{ name: "tokens_reasoning", type: "long", required: false },
{ name: "tokens_cache_read", type: "long", required: false },
{ name: "tokens_cache_write_5m", type: "long", required: false },
{ name: "tokens_cache_write_1h", type: "long", required: false },
{ name: "cost_input_microcents", type: "long", required: false },
{ name: "cost_output_microcents", type: "long", required: false },
{ name: "cost_cache_read_microcents", type: "long", required: false },
{ name: "cost_cache_write_microcents", type: "long", required: false },
{ name: "cost_total_microcents", type: "long", required: false },
{ name: "cost_input", type: "long", required: false },
{ name: "cost_output", type: "long", required: false },
{ name: "cost_cache_read", type: "long", required: false },
{ name: "cost_cache_write_5m", type: "long", required: false },
{ name: "cost_cache_write_1h", type: "long", required: false },
{ name: "cost_total", type: "long", required: false },
],
},
},
},
},
{ deleteBeforeReplace: $app.stage !== "production", ignoreChanges: ["metadata"] },
)
// Keep the deployed names and arguments: the R2 stats sync still uses this VPC
// and cluster after the legacy lake is removed.
const lakeVpc = new sst.aws.Vpc("LakeVpc")
const lakeCluster = new sst.aws.Cluster("LakeCluster", { vpc: lakeVpc })
////////////////
// DATABASE