Command Cookbook

Tested commands for common LakeXpress scenarios. Linux syntax shown; on Windows PowerShell, replace \ with ` and ./LakeXpress with .\LakeXpress.exe.

Table of Contents

Database Lifecycle Commands

# Initialize the logging database schema
./LakeXpress logdb init -a credentials.json --log_db_auth_id log_db_ms

# Drop the logging database schema (requires --confirm)
./LakeXpress logdb drop -a credentials.json --log_db_auth_id log_db_ms --confirm

# Clear all data, keep the schema
./LakeXpress logdb truncate -a credentials.json --log_db_auth_id log_db_ms

# Show locked tables (incremental syncs only)
./LakeXpress logdb locks -a credentials.json --log_db_auth_id log_db_ms

# Release stale or stuck locks
./LakeXpress logdb release-locks -a credentials.json --log_db_auth_id log_db_ms --confirm

Different Log Database Types

# SQL Server as log database
./LakeXpress logdb init -a credentials.json --log_db_auth_id log_db_ms_02

# PostgreSQL as log database
./LakeXpress logdb init -a credentials.json --log_db_auth_id log_db_pg

# DuckDB as log database
./LakeXpress logdb init -a credentials.json --log_db_auth_id log_db_duckdb

# MySQL as log database
./LakeXpress logdb init -a credentials.json --log_db_auth_id mysql_log_01

# SQLite as log database
./LakeXpress logdb init -a credentials.json --log_db_auth_id sqlite_01

Configuration Management

# Create a new sync configuration
./LakeXpress config create [OPTIONS]

# List all sync configurations
./LakeXpress config list -a credentials.json --log_db_auth_id log_db_ms

# Delete a sync configuration
./LakeXpress config delete -a credentials.json --log_db_auth_id log_db_ms --sync_id <SYNC_ID>

# Run from a legacy YAML config file
./LakeXpress run -c config_20251222.yml

Local Exports

PostgreSQL Source

PostgreSQL to Local (SQL Server log DB)

./LakeXpress config create \
    -a credentials.json \
    --log_db_auth_id log_db_ms_02 \
    --source_db_auth_id ds_04_pg \
    --source_db_name tpch \
    --source_schema_name tpch_1 \
    --fastbcp_dir_path ./FastBCP_linux-x64/latest/ \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --output_dir /tmp/tpch/ \
    --generate_metadata

./LakeXpress sync

SQL Server Source

SQL Server to Local (PostgreSQL log DB)

./LakeXpress config create \
    -a credentials.json \
    --log_db_auth_id log_db_pg \
    --source_db_auth_id ds_10_ms \
    --source_db_name adventureworksdw \
    --source_schema_name dbo \
    --fastbcp_dir_path ./FastBCP_linux-x64/latest/ \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --output_dir /tmp/tpch/ \
    --generate_metadata

./LakeXpress sync

SQL Server to Local (SQLite log DB)

./LakeXpress config create \
    -a credentials.json \
    --log_db_auth_id sqlite_01 \
    --source_db_auth_id ds_10_ms \
    --source_db_name adventureworksdw \
    --source_schema_name dbo \
    --fastbcp_dir_path ./FastBCP_linux-x64/latest/ \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --output_dir /tmp/tpch/ \
    --generate_metadata

./LakeXpress sync

SQL Server to Local (DuckDB log DB)

./LakeXpress logdb init -a credentials.json --log_db_auth_id log_db_duckdb

./LakeXpress config create \
    -a credentials.json \
    --log_db_auth_id log_db_duckdb \
    --source_db_auth_id ds_10_ms \
    --source_db_name adventureworksdw \
    --source_schema_name dbo \
    --fastbcp_dir_path ./FastBCP_linux-x64/latest/ \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --output_dir /tmp/tpch/ \
    --generate_metadata

./LakeXpress sync

SQL Server to Local (MySQL log DB)

./LakeXpress config create \
    -a credentials.json \
    --log_db_auth_id mysql_log_01 \
    --source_db_auth_id ds_10_ms \
    --source_db_name adventureworksdw \
    --source_schema_name dbo \
    --fastbcp_dir_path ./FastBCP_linux-x64/latest/ \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --output_dir /tmp/tpch/ \
    --generate_metadata

./LakeXpress sync

MySQL Source

MySQL to Local (SQL Server log DB)

./LakeXpress config create \
    -a credentials.json \
    --log_db_auth_id log_db_ms_02 \
    --source_db_auth_id mysql_01 \
    --source_db_name tpch \
    --source_schema_name tpch \
    --fastbcp_dir_path ./FastBCP_linux-x64/latest/ \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --output_dir /tmp/tpch/ \
    --generate_metadata

./LakeXpress sync

MariaDB Source

MariaDB to Local (SQL Server log DB)

./LakeXpress config create \
    -a credentials.json \
    --log_db_auth_id log_db_ms_02 \
    --source_db_auth_id maria_01 \
    --source_db_name tpch_mariadb \
    --source_schema_name tpch_mariadb \
    --fastbcp_dir_path ./FastBCP_linux-x64/latest/ \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --output_dir /tmp/tpch/ \
    --generate_metadata

./LakeXpress sync

AWS S3 Exports

PostgreSQL to S3

./LakeXpress config create \
    -a credentials.json \
    --log_db_auth_id log_db_ms_02 \
    --source_db_auth_id ds_04_pg \
    --source_db_name tpch \
    --source_schema_name tpch_1 \
    --fastbcp_dir_path ./FastBCP_linux-x64/latest/ \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --target_storage_id aws_s3_01 \
    --generate_metadata

./LakeXpress sync

PostgreSQL to S3 with Sub-Path and Table Filter

./LakeXpress config create \
    -a credentials.json \
    --log_db_auth_id log_db_ms_02 \
    --source_db_auth_id ds_04_pg \
    --source_db_name tpch \
    --source_schema_name "tpch_1%" \
    --fastbcp_dir_path ./FastBCP_linux-x64/latest/ \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --target_storage_id aws_s3_01 \
    --sub_path staging/data \
    --include "nation%" \
    --generate_metadata

./LakeXpress sync

MySQL to S3

./LakeXpress config create \
    -a credentials.json \
    --log_db_auth_id log_db_ms_02 \
    --source_db_auth_id mysql_01 \
    --source_db_name tpch \
    --source_schema_name tpch \
    --fastbcp_dir_path ./FastBCP_linux-x64/latest/ \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --target_storage_id aws_s3_01 \
    --sub_path mysql_export \
    --include "nation%" \
    --generate_metadata

./LakeXpress sync

MariaDB to S3

./LakeXpress config create \
    -a credentials.json \
    --log_db_auth_id log_db_ms_02 \
    --source_db_auth_id maria_01 \
    --source_db_name tpch_mariadb \
    --source_schema_name tpch_mariadb \
    --fastbcp_dir_path ./FastBCP_linux-x64/latest/ \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --target_storage_id aws_s3_01 \
    --sub_path mariadb_export \
    --include "nation%" \
    --generate_metadata

./LakeXpress sync

Azure Storage Exports

PostgreSQL to Azure

./LakeXpress config create \
    -a credentials.json \
    --log_db_auth_id log_db_ms_02 \
    --source_db_auth_id ds_04_pg \
    --source_db_name tpch \
    --source_schema_name tpch_1 \
    --fastbcp_dir_path ./FastBCP_linux-x64/latest/ \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --target_storage_id azure_01 \
    --generate_metadata

./LakeXpress sync

SQL Server to Azure

./LakeXpress config create \
    -a credentials.json \
    --log_db_auth_id log_db_pg \
    --source_db_auth_id ds_10_ms \
    --source_db_name adventureworksdw \
    --source_schema_name dbo \
    --fastbcp_dir_path ./FastBCP_linux-x64/latest/ \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --target_storage_id azure_01 \
    --generate_metadata

./LakeXpress sync

Google Cloud Storage Exports

PostgreSQL to GCS

./LakeXpress config create \
    -a credentials.json \
    --log_db_auth_id log_db_ms_02 \
    --source_db_auth_id ds_04_pg \
    --source_db_name tpch \
    --source_schema_name tpch_1 \
    --fastbcp_dir_path ./FastBCP_linux-x64/latest/ \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --target_storage_id gcs_01 \
    --generate_metadata

./LakeXpress sync

OneLake Exports

PostgreSQL to OneLake

./LakeXpress config create \
    -a credentials.json \
    --log_db_auth_id log_db_ms_02 \
    --source_db_auth_id ds_04_pg \
    --source_db_name tpch \
    --source_schema_name tpch_1 \
    --fastbcp_dir_path ./FastBCP_linux-x64/latest/ \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --target_storage_id fabric_onelake \
    --generate_metadata

./LakeXpress sync

Snowflake Publishing

Export to S3, then create Snowflake tables.

Snowflake External Tables

Data stays in S3; queryable via Snowflake.

External Tables with Views (Default)

./LakeXpress config create \
    -a credentials.json \
    --log_db_auth_id log_db_ms_02 \
    --source_db_auth_id ds_04_pg \
    --source_db_name tpch \
    --source_schema_name "tpch_1%" \
    --fastbcp_dir_path ./FastBCP_linux-x64/latest/ \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --target_storage_id aws_s3_01 \
    --generate_metadata \
    --sub_path snowflake_ext \
    --include "nation%" \
    --publish_target snowflake_pat

./LakeXpress sync

External Tables without Views

./LakeXpress config create \
    -a credentials.json \
    --log_db_auth_id log_db_ms_02 \
    --source_db_auth_id ds_04_pg \
    --source_db_name tpch \
    --source_schema_name "tpch_1%" \
    --fastbcp_dir_path ./FastBCP_linux-x64/latest/ \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --target_storage_id aws_s3_01 \
    --generate_metadata \
    --sub_path snowflake_ext \
    --include "nation%" \
    --publish_target snowflake_pat \
    --no_views

./LakeXpress sync

Snowflake Internal Tables

Data loaded into Snowflake storage.

Basic Internal Tables

./LakeXpress config create \
    -a credentials.json \
    --log_db_auth_id log_db_ms_02 \
    --source_db_auth_id ds_04_pg \
    --source_db_name tpch \
    --source_schema_name "tpch_1%" \
    --fastbcp_dir_path ./FastBCP_linux-x64/latest/ \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --target_storage_id aws_s3_01 \
    --generate_metadata \
    --sub_path snowflake_int \
    --include "nation%" \
    --publish_method internal \
    --publish_schema_pattern "INT_{schema}" \
    --publish_target snowflake_pat

./LakeXpress sync

Internal Tables with Custom Naming

./LakeXpress config create \
    -a credentials.json \
    --log_db_auth_id log_db_ms_02 \
    --source_db_auth_id ds_04_pg \
    --source_db_name tpch \
    --source_schema_name tpch_1 \
    --fastbcp_dir_path ./FastBCP_linux-x64/latest/ \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --target_storage_id aws_s3_01 \
    --generate_metadata \
    --sub_path production \
    --publish_method internal \
    --publish_schema_pattern "INT_{subpath}_{date}" \
    --publish_table_pattern "{schema}_{table}" \
    --publish_target snowflake_pat

./LakeXpress sync

Internal Tables with Primary Key Constraints

./LakeXpress config create \
    -a credentials.json \
    --log_db_auth_id log_db_ms_02 \
    --source_db_auth_id ds_04_pg \
    --source_db_name tpch \
    --source_schema_name tpch_1 \
    --fastbcp_dir_path ./FastBCP_linux-x64/latest/ \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --target_storage_id aws_s3_01 \
    --generate_metadata \
    --sub_path production \
    --publish_method internal \
    --publish_schema_pattern "INT_{subpath}_{date}" \
    --publish_table_pattern "{schema}_{table}" \
    --publish_target snowflake_pat \
    --snowflake_pk_constraints

./LakeXpress sync

SQL Server to Snowflake Internal

./LakeXpress config create \
    -a credentials.json \
    --log_db_auth_id log_db_ms_02 \
    --source_db_auth_id ds_10_ms \
    --source_db_name adventureworksdw \
    --source_schema_name dbo \
    --fastbcp_dir_path ./FastBCP_linux-x64/latest/ \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --target_storage_id aws_s3_01 \
    --publish_method internal \
    --publish_schema_pattern "adventureworksdw" \
    --publish_table_pattern "{table}" \
    --publish_target snowflake_pat

./LakeXpress sync

MySQL to Snowflake (External)

./LakeXpress config create \
    -a credentials.json \
    --log_db_auth_id log_db_ms_02 \
    --source_db_auth_id mysql_01 \
    --source_db_name tpch \
    --source_schema_name tpch \
    --fastbcp_dir_path ./FastBCP_linux-x64/latest/ \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --target_storage_id aws_s3_01 \
    --generate_metadata \
    --sub_path mysql_data \
    --include "nation%" \
    --publish_target snowflake_pat

./LakeXpress sync

MySQL to Snowflake (Internal)

./LakeXpress config create \
    -a credentials.json \
    --log_db_auth_id log_db_ms_02 \
    --source_db_auth_id mysql_01 \
    --source_db_name tpch \
    --source_schema_name tpch \
    --fastbcp_dir_path ./FastBCP_linux-x64/latest/ \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --target_storage_id aws_s3_01 \
    --generate_metadata \
    --sub_path mysql_data \
    --include "nation%" \
    --publish_method internal \
    --publish_schema_pattern "INT_{schema}" \
    --publish_target snowflake_pat

./LakeXpress sync

MariaDB to Snowflake (External)

./LakeXpress config create \
    -a credentials.json \
    --log_db_auth_id log_db_ms_02 \
    --source_db_auth_id maria_01 \
    --source_db_name tpch_mariadb \
    --source_schema_name tpch_mariadb \
    --fastbcp_dir_path ./FastBCP_linux-x64/latest/ \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --target_storage_id aws_s3_01 \
    --generate_metadata \
    --sub_path mariadb_data \
    --include "nation%" \
    --publish_target snowflake_pat

./LakeXpress sync

MariaDB to Snowflake (Internal)

./LakeXpress config create \
    -a credentials.json \
    --log_db_auth_id log_db_ms_02 \
    --source_db_auth_id maria_01 \
    --source_db_name tpch_mariadb \
    --source_schema_name tpch_mariadb \
    --fastbcp_dir_path ./FastBCP_linux-x64/latest/ \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --target_storage_id aws_s3_01 \
    --generate_metadata \
    --sub_path mariadb_data \
    --include "nation%" \
    --publish_method internal \
    --publish_schema_pattern "INT_{schema}" \
    --publish_target snowflake_pat

./LakeXpress sync

Databricks Publishing

Export to S3, then create Databricks Unity Catalog tables.

PostgreSQL to Databricks (External)

./LakeXpress config create \
    -a credentials.json \
    --log_db_auth_id log_db_ms_02 \
    --source_db_auth_id ds_04_pg \
    --source_db_name tpch \
    --source_schema_name tpch_1 \
    --fastbcp_dir_path ./FastBCP_linux-x64/latest/ \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --target_storage_id aws_s3_02 \
    --publish_target databricks_01 \
    --publish_method external

./LakeXpress sync

PostgreSQL to Databricks (Internal/Managed Delta Tables)

./LakeXpress config create \
    -a credentials.json \
    --log_db_auth_id log_db_ms_02 \
    --source_db_auth_id ds_04_pg \
    --source_db_name tpch \
    --source_schema_name tpch_1 \
    --fastbcp_dir_path ./FastBCP_linux-x64/latest/ \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --target_storage_id aws_s3_02 \
    --publish_target databricks_01 \
    --publish_method internal

./LakeXpress sync

SQL Server to Databricks (Internal)

./LakeXpress config create \
    -a credentials.json \
    --log_db_auth_id log_db_ms_02 \
    --source_db_auth_id ds_10_ms \
    --source_db_name adventureworksdw \
    --source_schema_name dbo \
    --fastbcp_dir_path ./FastBCP_linux-x64/latest/ \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --target_storage_id aws_s3_02 \
    --publish_target databricks_01 \
    --publish_schema_pattern "adventureworksdw" \
    --publish_method internal

./LakeXpress sync

AWS Glue Publishing

Export to S3, then register tables in AWS Glue Data Catalog.

PostgreSQL to AWS Glue

./LakeXpress config create \
    -a credentials.json \
    --log_db_auth_id log_db_ms_02 \
    --source_db_auth_id ds_04_pg \
    --source_db_name tpch \
    --source_schema_name tpch_1 \
    --target_storage_id aws_s3_01 \
    --fastbcp_dir_path ./FastBCP_linux-x64/latest/ \
    --fastbcp_p 4 \
    --n_jobs 2 \
    --publish_target glue_01 \
    --publish_schema_pattern "lakexpress_{schema}" \
    --publish_table_pattern "{table}"

./LakeXpress sync

PostgreSQL to AWS Glue with Custom FastBCP Config

./LakeXpress config create \
    -a credentials.json \
    --log_db_auth_id log_db_ms_02 \
    --source_db_auth_id ds_04_pg \
    --source_db_name tpch \
    --source_schema_name tpch_1 \
    --target_storage_id aws_s3_01 \
    --fastbcp_dir_path ./FastBCP_linux-x64/latest/ \
    --fastbcp_p 2 \
    --fastbcp_table_config "lineitem:DataDriven:TO_CHAR(l_shipdate, 'YYYY'):8;orders:Ctid::4" \
    --n_jobs 2 \
    --publish_target glue_01 \
    --publish_schema_pattern "lakexpress_{schema}" \
    --publish_table_pattern "{table}"

./LakeXpress sync

SQL Server to AWS Glue

./LakeXpress config create \
    -a credentials.json \
    --log_db_auth_id log_db_ms_02 \
    --source_db_auth_id ds_09_ms \
    --source_db_name tpch \
    --source_schema_name tpch_1 \
    --target_storage_id aws_s3_01 \
    --fastbcp_dir_path ./FastBCP_linux-x64/latest/ \
    --fastbcp_p 4 \
    --n_jobs 2 \
    --publish_target glue_01 \
    --publish_schema_pattern "lakexpress_{schema}" \
    --publish_table_pattern "{table}"

./LakeXpress sync

SQL Server to AWS Glue with Custom FastBCP Config

./LakeXpress config create \
    -a credentials.json \
    --log_db_auth_id log_db_ms_02 \
    --source_db_auth_id ds_09_ms \
    --source_db_name tpch \
    --source_schema_name tpch_1 \
    --target_storage_id aws_s3_01 \
    --fastbcp_dir_path ./FastBCP_linux-x64/latest/ \
    --fastbcp_p 2 \
    --fastbcp_table_config "lineitem:DataDriven:YEAR(l_shipdate):8;orders:Physloc::4" \
    --n_jobs 2 \
    --publish_target glue_01 \
    --publish_database_name "lakexpress_tpch" \
    --publish_table_pattern "{schema}_{table}"

./LakeXpress sync

SQL Server AdventureWorks to AWS Glue

./LakeXpress config create \
    -a credentials.json \
    --log_db_auth_id log_db_ms_02 \
    --source_db_auth_id ds_10_ms \
    --source_db_name adventureworksdw \
    --source_schema_name dbo \
    --fastbcp_dir_path ./FastBCP_linux-x64/latest/ \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --target_storage_id aws_s3_02 \
    --publish_target glue_01 \
    --publish_schema_pattern "adventureworksdw"

./LakeXpress sync

Microsoft Fabric Publishing

Export to OneLake, then create Fabric Lakehouse tables.

PostgreSQL to Fabric (Internal/Delta Tables)

./LakeXpress config create \
    -a credentials.json \
    --log_db_auth_id log_db_ms_02 \
    --source_db_auth_id ds_04_pg \
    --source_db_name tpch \
    --source_schema_name tpch_1 \
    --fastbcp_dir_path ./FastBCP_linux-x64/latest/ \
    --target_storage_id fabric_onelake \
    --publish_target fabric_01 \
    --publish_method internal \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --generate_metadata

./LakeXpress sync

PostgreSQL to Fabric (External/SQL Views)

./LakeXpress config create \
    -a credentials.json \
    --log_db_auth_id log_db_ms_02 \
    --source_db_auth_id ds_04_pg \
    --source_db_name tpch \
    --source_schema_name tpch_1 \
    --fastbcp_dir_path ./FastBCP_linux-x64/latest/ \
    --target_storage_id fabric_onelake \
    --publish_target fabric_01 \
    --publish_method external \
    --publish_table_pattern "VW_{schema}_{table}" \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --generate_metadata

./LakeXpress sync

Incremental Sync

Watermark-based delta exports.

Basic Incremental Sync

./LakeXpress config create \
    -a credentials.json \
    --log_db_auth_id log_db_ms_02 \
    --source_db_auth_id ds_04_pg \
    --source_db_name tpch \
    --source_schema_name tpch_1_incremental \
    --fastbcp_dir_path ./FastBCP_linux-x64/latest/ \
    --target_storage_id aws_s3_01 \
    --incremental_table "tpch_1_incremental.lineitem:l_shipdate:date" \
    --incremental_table "tpch_1_incremental.orders:o_orderdate:date" \
    --generate_metadata

# First sync exports all data; subsequent syncs export only new/changed data
./LakeXpress sync

Debug Mode

./LakeXpress sync --log_lev DEBUG

See Also