Example datapackage¶

name: datapackage_example

resources:
  # API extraction
  - name: invoices
    path: data/invoices.csv
    sources:   # Request settings
      - method: get   # required (e.g. get, post)
        path: https://api.example.com   # required
        timeout: 30   # optional (Defaults to 30 seconds)
        params: {}   # optional
        headers: {}   # optional
        stream: false   # optional (Defaults to false)
    dpetl_extract:
      mode: api

  # CLI extraction
  - name: external_data
    path: data/external_data.csv
    dpetl_extract:
      mode: cli
      arguments:   # required. List of shell commands to run sequentially
        - curl -o data/external_data.csv https://api.example.com/export
        - python scripts/clean_data.py data/external_data.csv

  # Email extraction
  - name: payroll_from_email
    path: data/payroll.xlsx
    dpetl_extract:
      mode: email
      mailbox: INBOX   # optional (Defaults to INBOX)
      criteria:   # optional
        subject: "Payroll Report"   # optional (Defaults to resource name. See also the flag --add-package-name)
        from_: "finance@example.com"   # optional
        date_gte: 2024-01-01   # optional (See also the flag --today-email)

  # Column renaming
  - name: customers
    path: data/customers.csv
    schema:
      fields:
        - name: Name
          type: string
          target: customer_name
        - name: Region
          type: string
          target: customer_region
    dpetl_transform:
      format: csv.gz   # optional (Defaults to csv.gz)
      path: data/processed   # optional (Defaults to data)
      encoding: utf-8   # optional (Defaults to utf-8)
      delimiter: ';'   # optional (Defaults to ,)

  # Anonymization of sensitive fields
  - name: employees
    path: data/employees.csv
    schema:
      fields:
        - name: employee_name
          type: string
          target: name_hashed
          anonymize:
            method: '[#***]'
        - name: tax_id
          type: string
          target: tax_id_encrypted
          anonymize:
            method: aes_siv
            context: department
            annotation: TAX:11
            filter: 'department == "Finance"'
        - name: phone
          type: string
          anonymize:
            method: '[###-***-####]'

  # External command
  - name: payroll
    path: data/payroll.csv
    dpetl_transform:
      cli:
        arguments:   # required. List of shell commands to run sequentially
          - python scripts/clean_payroll.py
        path: data/processed   # optional (Defaults to dpetl_transform.path)
        pre_process: true   # optional (Defaults to true)
        stdin: true   # optional (Defaults to false)

# Package level. Passed to every cli command as an environment variable
custom:
  - year: 2027
    region: south

# Load configuration (defined once per package)
dpetl_load:
  owner: github-username
  repo: my-data-repo
  level: user   # optional (Defaults to user)
  visibility: private   # optional (Defaults to private)