Help optimize this configuration file, including obtaining project names through tokens and automatically adding project labels.

[liao-bo]

Environment Information

OpenTelemetry Collector Contrib Version: v0.130.0

Deployment Environment: Bare Metal

Data Volume: Approximately 1000 spans/second, 500 metrics/second

Hardware Configuration: 4 CPU cores, 8GB RAM

Current Configuration

exporters:
  debug: {}
  loki:
    endpoint: http://172.31.7.136:30376/loki/api/v1/push
  otlp/tempo:
    endpoint: 172.31.7.136:31556
    tls:
      insecure: true
  otlphttp/langfuse-a:
    compression: none
    endpoint: http://172.31.7.136:31381/api/public/otel
    headers:
      #authorization: "Basic ****"
      authorization: "Basic ****"
    timeout: 30s
    tls:
      insecure: true

  otlphttp/langfuse-b:
    compression: none
    endpoint: http://172.31.7.136:31381/api/public/otel
    headers:
      authorization: "Basic ****"
    timeout: 30s
    tls:
      insecure: true
  prometheus:
    endpoint: 0.0.0.0:8889
    namespace: default
  prometheusremotewrite:
    endpoint: http://172.31.7.136:31716/api/v1/write
extensions:
  health_check:
    endpoint: ${env:MY_POD_IP}:13133
  bearertokenauth:
    tokens:
      - "token-for-account-a" 
      - "token-for-account-b"
processors:
  batch: {}
  filter/logs:
    logs:
      log_record:
      - IsMatch(attributes["code.file.path"], "^/otel-auto-instrumentation-.*")

  memory_limiter:
    check_interval: 5s
    limit_percentage: 80
    spike_limit_percentage: 25
  redaction:
    allow_all_keys: true
    blocked_key_patterns:
      - ".*api_key.*"
      - ".*auth_token" 
  
  transform/token-mapping:
    trace_statements:
      - context: span 
        statements:
          - set(attributes["project_"], "service-a") where resource.attributes["auth_token"] == "token-for-account-a"
          - set(attributes["project_"], "service-b") where resource.attributes["auth_token"] == "token-for-account-b"
          - delete_key(resource.attributes, "auth_token") 

    metric_statements:
      - context: datapoint
        statements:
          - set(attributes["project_"], "service-a") where resource.attributes["auth_token"] == "token-for-account-a"
          - set(attributes["project_"], "service-b") where resource.attributes["auth_token"] == "token-for-account-b"
          - delete_key(resource.attributes, "auth_token") 

    log_statements:
      - context: log
        statements:
          - set(attributes["project_"], "service-a") where resource.attributes["auth_token"] == "token-for-account-a"
          - set(attributes["project_"], "service-b") where resource.attributes["auth_token"] == "token-for-account-b"
          - delete_key(resource.attributes, "auth_token") 

connectors:
  routing:
    table:
      - context: span
        condition: attributes["project_"] == "service-a"
        pipelines: [ traces/langfuse-a ]
      - context: span
        condition: attributes["project_"] == "service-b"
        pipelines: [ traces/langfuse-b ]
  
receivers:
  jaeger:
    protocols:
      grpc:
        endpoint: ${env:MY_POD_IP}:14250
      thrift_compact:
        endpoint: ${env:MY_POD_IP}:6831
      thrift_http:
        endpoint: ${env:MY_POD_IP}:14268
  otlp:
    protocols:
      grpc:
        endpoint: ${env:MY_POD_IP}:4317
        auth:
          authenticator: bearertokenauth
        include_metadata: true
      http:
        endpoint: ${env:MY_POD_IP}:4318
        auth:
          authenticator: bearertokenauth
        include_metadata: true

  zipkin:
    endpoint: ${env:MY_POD_IP}:9411
service:
  extensions: 
  - health_check
  - bearertokenauth
  pipelines:
    logs:
      exporters:
      - loki
      processors:
      - memory_limiter
      - filter/logs
      - batch
      - transform/token-mapping
      - redaction
      receivers:
      - otlp
    metrics:
      exporters:
      - prometheus
      - prometheusremotewrite
      processors:
      - memory_limiter
      - batch
      - transform/token-mapping
      - redaction
      receivers:
      - otlp
      #- prometheus
    traces/in:
      receivers:
      - otlp
      processors:
      - memory_limiter
      - batch
      - transform/token-mapping
      - redaction
      exporters:
      - debug
      - routing
      - otlp/tempo
    traces/langfuse-a:
      receivers:
      - routing
      processors:
      - redaction
      exporters:
      - otlphttp/langfuse-a
    traces/langfuse-b:
      receivers:
      - routing
      processors:
      - redaction
      exporters:
      - otlphttp/langfuse-b

Question

The configration is working OK, but the configuration content is too complicated . If I have dozens of projects, my configuration entries will become too lengthy and unmaintainable. Can you help me optimize this configuration?

1. How can I get authenticate token from receivers-otpl grpc or http header and save it to resouces.attribute . Now the OTEL client send the token twice (header and resource_attributes ). The client command is like :

 opentelemetry-instrument --traces_exporter none  --metrics_exporter none --logs_exporter otlp,console  --exporter_otlp_endpoint http://172.31.2.178:4317 --exporter_otlp_headers "authorization=Bearer token-for-account-b" --service_name liaobo-local  --resource_attributes "auth_token=token-for-account-b" python app.py

Is it possible to transmit only in the HEADER and then have the Collector automatically save it in RESOURCE.ATTRIBUTES？

2. Could the transform/token-mapping plugin support a single token-to-project mapping configuration that applies globally across all telemetry signals (traces, logs, and metrics), instead of requiring separate configurations for each data type? This would greatly simplify the configuration .