mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-18 07:12:47 +03:00
chore(stats): retire legacy s3 lake (#52515)
This commit is contained in:
@@ -1,89 +0,0 @@
|
||||
// Retired in favor of the R2 lake. Nothing writes to it anymore, but the table,
|
||||
// catalog and Athena workgroup stay while the anomalyco/anomaly inference and
|
||||
// analytics dashboards still query its history. Pulumi reads forceDestroy and
|
||||
// retainOnDelete from state at delete time, so keep those settings until the
|
||||
// lake is removed.
|
||||
|
||||
const current = aws.getCallerIdentityOutput({})
|
||||
const partition = aws.getPartitionOutput({})
|
||||
const region = aws.getRegionOutput({})
|
||||
|
||||
const tableBucketName = `opencode-${$app.stage}-lake`
|
||||
const glueCatalogName = "s3tablescatalog"
|
||||
const s3TablesBucketWildcardArn = $interpolate`arn:${partition.partition}:s3tables:${region.region}:${current.accountId}:bucket/*`
|
||||
|
||||
export const tableBucket = new aws.s3tables.TableBucket("LakeTableBucket", {
|
||||
name: tableBucketName,
|
||||
forceDestroy: true,
|
||||
})
|
||||
|
||||
new aws.cloudcontrol.Resource(
|
||||
"LakeS3TablesCatalog",
|
||||
{
|
||||
typeName: "AWS::Glue::Catalog",
|
||||
desiredState: $jsonStringify({
|
||||
Name: glueCatalogName,
|
||||
Description: "Federated catalog for S3 Tables",
|
||||
FederatedCatalog: {
|
||||
Identifier: s3TablesBucketWildcardArn,
|
||||
ConnectionName: "aws:s3tables",
|
||||
},
|
||||
CreateDatabaseDefaultPermissions: [
|
||||
{
|
||||
Principal: {
|
||||
DataLakePrincipalIdentifier: "IAM_ALLOWED_PRINCIPALS",
|
||||
},
|
||||
Permissions: ["ALL"],
|
||||
},
|
||||
],
|
||||
CreateTableDefaultPermissions: [
|
||||
{
|
||||
Principal: {
|
||||
DataLakePrincipalIdentifier: "IAM_ALLOWED_PRINCIPALS",
|
||||
},
|
||||
Permissions: ["ALL"],
|
||||
},
|
||||
],
|
||||
AllowFullTableExternalDataAccess: "True",
|
||||
}),
|
||||
},
|
||||
{ dependsOn: [tableBucket] },
|
||||
)
|
||||
|
||||
const athenaResultsBucket = new aws.s3.Bucket(
|
||||
"LakeAthenaResults",
|
||||
{
|
||||
bucket: `opencode-${$app.stage}-lake-athena-results`,
|
||||
forceDestroy: true,
|
||||
},
|
||||
{ retainOnDelete: false },
|
||||
)
|
||||
|
||||
// Keep the archived delivery failures until their records have been reconciled.
|
||||
new aws.s3.Bucket(
|
||||
"LakeFirehoseErrors",
|
||||
{
|
||||
bucket: `opencode-${$app.stage}-lake-firehose-errors`,
|
||||
forceDestroy: true,
|
||||
},
|
||||
{ retainOnDelete: false },
|
||||
)
|
||||
|
||||
new aws.athena.Workgroup("LakeAthenaWorkgroup", {
|
||||
name: `opencode-${$app.stage}-lake-workgroup`,
|
||||
forceDestroy: true,
|
||||
configuration: {
|
||||
enforceWorkgroupConfiguration: true,
|
||||
publishCloudwatchMetricsEnabled: true,
|
||||
// Athena bills $5/TB scanned; kill any query that would scan more than 2 TB
|
||||
// so a regression cannot silently burn money. Stats sync full passes scan
|
||||
// ~250 GB as of 2026-07.
|
||||
bytesScannedCutoffPerQuery: 2 * 1024 ** 4,
|
||||
resultConfiguration: {
|
||||
outputLocation: $interpolate`s3://${athenaResultsBucket.bucket}/`,
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
export const lakeVpc = new sst.aws.Vpc("LakeVpc")
|
||||
export const lakeCluster = new sst.aws.Cluster("LakeCluster", { vpc: lakeVpc })
|
||||
+4
-87
@@ -1,93 +1,10 @@
|
||||
import { lakeCluster, tableBucket } from "./lake"
|
||||
import { EMAILOCTOPUS_API_KEY } from "./app"
|
||||
import { domain } from "./stage"
|
||||
|
||||
////////////////
|
||||
// LAKE
|
||||
////////////////
|
||||
|
||||
const inferenceNamespace = new aws.s3tables.Namespace("LakeInferenceNamespace", {
|
||||
namespace: "inference",
|
||||
tableBucketArn: tableBucket.arn,
|
||||
})
|
||||
|
||||
new aws.s3tables.Table(
|
||||
"LakeInferenceEventTable",
|
||||
{
|
||||
name: "event",
|
||||
namespace: inferenceNamespace.namespace,
|
||||
tableBucketArn: inferenceNamespace.tableBucketArn,
|
||||
format: "ICEBERG",
|
||||
metadata: {
|
||||
iceberg: {
|
||||
schema: {
|
||||
fields: [
|
||||
{ name: "event_timestamp", type: "string", required: false },
|
||||
{ name: "event_date", type: "string", required: false },
|
||||
{ name: "event_type", type: "string", required: false },
|
||||
{ name: "dataset", type: "string", required: false },
|
||||
{ name: "cf_continent", type: "string", required: false },
|
||||
{ name: "cf_country", type: "string", required: false },
|
||||
{ name: "cf_city", type: "string", required: false },
|
||||
{ name: "cf_region", type: "string", required: false },
|
||||
{ name: "cf_latitude", type: "double", required: false },
|
||||
{ name: "cf_longitude", type: "double", required: false },
|
||||
{ name: "cf_timezone", type: "string", required: false },
|
||||
{ name: "duration", type: "double", required: false },
|
||||
{ name: "request_length", type: "long", required: false },
|
||||
{ name: "status", type: "int", required: false },
|
||||
{ name: "ip", type: "string", required: false },
|
||||
{ name: "is_stream", type: "boolean", required: false },
|
||||
{ name: "session", type: "string", required: false },
|
||||
{ name: "request", type: "string", required: false },
|
||||
{ name: "client", type: "string", required: false },
|
||||
{ name: "user_agent", type: "string", required: false },
|
||||
{ name: "model", type: "string", required: false },
|
||||
{ name: "model_tier", type: "string", required: false },
|
||||
{ name: "model_variant", type: "string", required: false },
|
||||
{ name: "source", type: "string", required: false },
|
||||
{ name: "provider", type: "string", required: false },
|
||||
{ name: "provider_model", type: "string", required: false },
|
||||
{ name: "llm_error_code", type: "int", required: false },
|
||||
{ name: "llm_error_message", type: "string", required: false },
|
||||
{ name: "error_response", type: "string", required: false },
|
||||
{ name: "error_type", type: "string", required: false },
|
||||
{ name: "error_message", type: "string", required: false },
|
||||
{ name: "error_cause", type: "string", required: false },
|
||||
{ name: "error_cause2", type: "string", required: false },
|
||||
{ name: "api_key", type: "string", required: false },
|
||||
{ name: "workspace", type: "string", required: false },
|
||||
{ name: "user_id", type: "string", required: false },
|
||||
{ name: "is_subscription", type: "boolean", required: false },
|
||||
{ name: "subscription", type: "string", required: false },
|
||||
{ name: "response_length", type: "long", required: false },
|
||||
{ name: "time_to_first_byte", type: "long", required: false },
|
||||
{ name: "timestamp_first_byte", type: "long", required: false },
|
||||
{ name: "timestamp_last_byte", type: "long", required: false },
|
||||
{ name: "tokens_input", type: "long", required: false },
|
||||
{ name: "tokens_output", type: "long", required: false },
|
||||
{ name: "tokens_reasoning", type: "long", required: false },
|
||||
{ name: "tokens_cache_read", type: "long", required: false },
|
||||
{ name: "tokens_cache_write_5m", type: "long", required: false },
|
||||
{ name: "tokens_cache_write_1h", type: "long", required: false },
|
||||
{ name: "cost_input_microcents", type: "long", required: false },
|
||||
{ name: "cost_output_microcents", type: "long", required: false },
|
||||
{ name: "cost_cache_read_microcents", type: "long", required: false },
|
||||
{ name: "cost_cache_write_microcents", type: "long", required: false },
|
||||
{ name: "cost_total_microcents", type: "long", required: false },
|
||||
{ name: "cost_input", type: "long", required: false },
|
||||
{ name: "cost_output", type: "long", required: false },
|
||||
{ name: "cost_cache_read", type: "long", required: false },
|
||||
{ name: "cost_cache_write_5m", type: "long", required: false },
|
||||
{ name: "cost_cache_write_1h", type: "long", required: false },
|
||||
{ name: "cost_total", type: "long", required: false },
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{ deleteBeforeReplace: $app.stage !== "production", ignoreChanges: ["metadata"] },
|
||||
)
|
||||
// Keep the deployed names and arguments: the R2 stats sync still uses this VPC
|
||||
// and cluster after the legacy lake is removed.
|
||||
const lakeVpc = new sst.aws.Vpc("LakeVpc")
|
||||
const lakeCluster = new sst.aws.Cluster("LakeCluster", { vpc: lakeVpc })
|
||||
|
||||
////////////////
|
||||
// DATABASE
|
||||
|
||||
Reference in New Issue
Block a user