feat: sandbox retention basic settings (#29842)

api/.env.example

@@ -690,3 +690,8 @@ ANNOTATION_IMPORT_RATE_LIMIT_PER_MINUTE=5
 ANNOTATION_IMPORT_RATE_LIMIT_PER_HOUR=20
 # Maximum number of concurrent annotation import tasks per tenant
 ANNOTATION_IMPORT_MAX_CONCURRENT=5
+
+# Sandbox expired records clean configuration
+SANDBOX_EXPIRED_RECORDS_CLEAN_GRACEFUL_PERIOD=21
+SANDBOX_EXPIRED_RECORDS_CLEAN_BATCH_SIZE=1000
+SANDBOX_EXPIRED_RECORDS_RETENTION_DAYS=30

api/README.md

@@ -84,7 +84,7 @@
 1. If you need to handle and debug the async tasks (e.g. dataset importing and documents indexing), please start the worker service.
 
 ```bash
-uv run celery -A app.celery worker -P threads -c 2 --loglevel INFO -Q dataset,priority_dataset,priority_pipeline,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_executor
+uv run celery -A app.celery worker -P threads -c 2 --loglevel INFO -Q dataset,priority_dataset,priority_pipeline,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_executor,retention
 ```
 
 Additionally, if you want to debug the celery scheduled tasks, you can run the following command in another terminal to start the beat service:

api/configs/feature/__init__.py

@@ -1270,6 +1270,21 @@ class TenantIsolatedTaskQueueConfig(BaseSettings):
     )
 
 
+class SandboxExpiredRecordsCleanConfig(BaseSettings):
+    SANDBOX_EXPIRED_RECORDS_CLEAN_GRACEFUL_PERIOD: NonNegativeInt = Field(
+        description="Graceful period in days for sandbox records clean after subscription expiration",
+        default=21,
+    )
+    SANDBOX_EXPIRED_RECORDS_CLEAN_BATCH_SIZE: PositiveInt = Field(
+        description="Maximum number of records to process in each batch",
+        default=1000,
+    )
+    SANDBOX_EXPIRED_RECORDS_RETENTION_DAYS: PositiveInt = Field(
+        description="Retention days for sandbox expired workflow_run records and message records",
+        default=30,
+    )
+
+
 class FeatureConfig(
     # place the configs in alphabet order
     AppExecutionConfig,
@@ -1295,6 +1310,7 @@ class FeatureConfig(
     PositionConfig,
     RagEtlConfig,
     RepositoryConfig,
+    SandboxExpiredRecordsCleanConfig,
     SecurityConfig,
     TenantIsolatedTaskQueueConfig,
     ToolConfig,

api/docker/entrypoint.sh

@@ -34,10 +34,10 @@ if [[ "${MODE}" == "worker" ]]; then
   if [[ -z "${CELERY_QUEUES}" ]]; then
     if [[ "${EDITION}" == "CLOUD" ]]; then
       # Cloud edition: separate queues for dataset and trigger tasks
-      DEFAULT_QUEUES="dataset,priority_dataset,priority_pipeline,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow_professional,workflow_team,workflow_sandbox,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_executor"
+      DEFAULT_QUEUES="dataset,priority_dataset,priority_pipeline,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow_professional,workflow_team,workflow_sandbox,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_executor,retention"
     else
       # Community edition (SELF_HOSTED): dataset, pipeline and workflow have separate queues
-      DEFAULT_QUEUES="dataset,priority_dataset,priority_pipeline,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_executor"
+      DEFAULT_QUEUES="dataset,priority_dataset,priority_pipeline,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_executor,retention"
     fi
   else
     DEFAULT_QUEUES="${CELERY_QUEUES}"
@@ -69,6 +69,53 @@ if [[ "${MODE}" == "worker" ]]; then
 
 elif [[ "${MODE}" == "beat" ]]; then
   exec celery -A app.celery beat --loglevel ${LOG_LEVEL:-INFO}
+
+elif [[ "${MODE}" == "job" ]]; then
+  # Job mode: Run a one-time Flask command and exit
+  # Pass Flask command and arguments via container args
+  # Example K8s usage:
+  #   args:
+  #   - create-tenant
+  #   - --email
+  #   - admin@example.com
+  #
+  # Example Docker usage:
+  #   docker run -e MODE=job dify-api:latest create-tenant --email admin@example.com
+
+  if [[ $# -eq 0 ]]; then
+    echo "Error: No command specified for job mode."
+    echo ""
+    echo "Usage examples:"
+    echo "  Kubernetes:"
+    echo "    args: [create-tenant, --email, admin@example.com]"
+    echo ""
+    echo "  Docker:"
+    echo "    docker run -e MODE=job dify-api create-tenant --email admin@example.com"
+    echo ""
+    echo "Available commands:"
+    echo "  create-tenant, reset-password, reset-email, upgrade-db,"
+    echo "  vdb-migrate, install-plugins, and more..."
+    echo ""
+    echo "Run 'flask --help' to see all available commands."
+    exit 1
+  fi
+
+  echo "Running Flask job command: flask $*"
+  
+  # Temporarily disable exit on error to capture exit code
+  set +e
+  flask "$@"
+  JOB_EXIT_CODE=$?
+  set -e
+
+  if [[ ${JOB_EXIT_CODE} -eq 0 ]]; then
+    echo "Job completed successfully."
+  else
+    echo "Job failed with exit code ${JOB_EXIT_CODE}."
+  fi
+
+  exit ${JOB_EXIT_CODE}
+
 else
   if [[ "${DEBUG}" == "true" ]]; then
     exec flask run --host=${DIFY_BIND_ADDRESS:-0.0.0.0} --port=${DIFY_PORT:-5001} --debug