Add Data Source Tutorial

In this article, you'll learn:

• How to list available data source connectors within a system
• Where to find parameters required for a given data source
• How to test a connection to a data source
• How to add a data source using the BigID API

Scenario

You have a custom system that maintains an inventory of your databases and have just purchased BigID to better understand what’s inside those databases for an internal AI initiative. Your executive staff says this will “bring your organization into the modern age”. Use the BigID API to add a data source to start your integration with the inventory system.

In this tutorial, we’ll use SAMPLE as our session token. This is unique to the training sandbox and will not work in other environments. See BigID API/Tutorial for information on authenticating with BigID.

Discovering Data Sources

You can see what data source connectors are installed in your environment through the BigID UI, but since we’re focused on the API (and because all actions in the UI can be performed in the API), we are going to use the API to retrieve them.

Press Send on the request below to get a listing of the data source connectors installed on our test BigID system.

You’ll see our test system has around 70 different data source connectors installed. If you don’t see a data source you want to use, you can develop your own or it might already exist, just not on our system. See the BigID docs for an exhaustive list.

Getting Data Source Parameters

Each type of data source has different parameters needed to connect to it. These parameters can be as simple as a username and password or as complex as rate limiting information. BigID uses templates to display these fields to the user in the UI. We can use those same templates to determine what we need to supply when adding a data source via the API. We’re going to add a MySQL database. Use the below request to get the template for an rdb-mysql data source.

As you can see in the request above, there’s a ton of different options to customize how we connect to a MySQL database. For our purposes, we’re going to go with just the most basic options as seen below.

{
    "name": "rdb-mysql",
    ...
    "fields": [
        {
            "type": "string",
            "name": "name",
            "apiName": "name",
            "displayName": "Data Source Name",
            "placeholder": "Type data source name",
            "mandatory": true,
            "mandatoryForTest": true,
            "validation": [
                {
                    "regex": "^[\\w\\-\\s\\(\\):]+$",
                    "errorText": "Invalid value. Please use alphanumeric characters, spaces, underscore, dash and parentheses."
                }
            ],
            "section": "connection",
            "order": 0,
            "enabled": true
        },
        {
            "type": "stringSelect",
            "name": "enabled",
            "apiName": "enabled",
            "displayName": "Status",
            "defaultValue": "yes",
            "options": [
                {
                    "value": "yes",
                    "label": "Enabled"
                },
                {
                    "value": "no",
                    "label": "Disabled"
                }
            ],
            "section": "connection",
            "order": 1,
            "enabled": true
        },
        {
            "type": "string",
            "name": "dbUrl",
            "apiName": "rdb_url",
            "displayName": "DB URL",
            "placeholder": ":",
            "tooltipText": "Enter a connection string to the data source.",
            "section": "connection",
            "mandatoryForTest": true,
            "order": 0,
            "enabled": true
        },
        {
            "type": "string",
            "name": "dBSchemaName",
            "apiName": "rdb_name",
            "displayName": "DB/Schema Name",
            "placeholder": ".",
            "tooltipText": "Enter database or schema name. Note: this field may be case sensitive depending on the specific data source.",
            "isSeparatorAfter": true,
            "mandatoryForTest": false,
            "section": "connection",
            "order": 1,
            "enabled": true
        },
        {
            "type": "string",
            "name": "userName",
            "apiName": "username",
            "displayName": "User Name",
            "visibleIf": [
                {
                    "field": "useCredentialOrNamePass",
                    "value": false
                }
            ],
            "enabledIf": [
                {
                    "field": "useCredentialOrNamePass",
                    "value": false
                }
            ],
            "mandatoryForTest": true,
            "section": "connection",
            "order": 6,
            "enabled": true
        },
        {
            "type": "password",
            "name": "password",
            "displayName": "Password",
            "apiName": "password",
            "visibleIf": [
                {
                    "field": "useCredentialOrNamePass",
                    "value": false
                }
            ],
            "enabledIf": [
                {
                    "field": "useCredentialOrNamePass",
                    "value": false
                }
            ],
            "mandatoryForTest": true,
            "isSeparatorAfter": true,
            "nullifyIfNotChanged": true,
            "section": "connection",
            "order": 7,
            "enabled": true
        },

        {
            "type": "string",
            "name": "type",
            "apiName": "type",
            "mandatory": true,
            "hidden": true,
            "defaultValue": "rdb-mysql",
            "section": "connection",
            "order": 9,
            "enabled": true
        }
        ...
    ]
}

Testing a data source

In any organization, getting the correct information to access a data source can be an arduous process. Especially after data source credentials have gone through multiple levels of your organization to make it to you. Because of this, we recommend testing any data source credentials before you enter them into BigID. This will also ensure BigID has proper network connectivity to the data source. You can do this with the /ds-connection-test endpoint.

Just be sure you include ‘isNewPassword’ to the request, otherwise BigID will attempt to test the existing data source in your system.

Adding a data source

Now that we know what parameters to pass, let’s create our data source. We just need to send a POST to the /ds_conenctions endpoint with our parameters. Let’s connect to the BigID test data set:

• Type: rdb-mysql
• URL: sql.mybigid.com
• Username: bigid
• Password: bigid111
• rdb_name: rockstream

Every data source in BigID also needs a unique name. For your data source, you should use the name RANDOMHERE so you don’t conflict with other users.

If we retrieve our data sources, now we should see a new data source with the information we supplied above. Use CTRL+F (or CMD+F) in your browser to find the data source you created in the request below.

Code Samples

Python

# Add Data Source Tutorial
import requests
import json

# Base URL of the BigID API (training sandbox)
base_url = "https://developer.bigid.com/api/v1"

# Session token (replace SAMPLE with actual session token)
headers = {
    "Authorization": "Bearer SAMPLE",
    "Content-Type": "application/json"
}

# 1. Get list of available data source connectors
url = f"{base_url}/ds-templates"
response = requests.get(url, headers=headers)
print(response.json())

# 2. Get the rdb-mysql template to find required fields
url = f"{base_url}/ds-templates/rdb-mysql"
response = requests.get(url, headers=headers)
print(response.json())

# 3. Test the connection to the example MySQL data source
url = f"{base_url}/ds-connection-test"
payload = {
        "ds_connection": {
            "username": "bigid",
            "password": "bigid111",
            "rdb_url": "sql.mybigid.com",
            "type": "rdb-mysql",
            "enabled": "yes",
            "rdb_name": "rockstream"
        },
        "isNewPassword": True
    }
response = requests.post(url, headers=headers, json=payload)
print(response.json())

# 4. Add the example MySQL data source
url = f"{base_url}/ds_connections"
payload = {
    "ds_connection": {
        "name": "645", # Name for every data source should be unique
        "type": "rdb-mysql",
        "rdb_url": "sql.mybigid.com",
        "username": "bigid",
        "password": "bigid111",
        "rdb_name": "rockstream"
    }
}
response = requests.post(url, headers=headers, json=payload)
print(response.json())

JavaScript

// Add Data Source tutorial
const BASE_URL = "https://developer.bigid.com/api/v1";
const HEADERS = {
  "Authorization": "Bearer SAMPLE", // Replace SAMPLE with a actual session token
  "Content-Type": "application/json"
};

// 1. Get list of available data source connectors
async function getDataSourceConnectors() {
  const res = await fetch(`${BASE_URL}/ds-templates`, {
    method: "GET",
    headers: HEADERS
  });
  const data = await res.json();
  console.log("Available connectors:", JSON.stringify(data, null, 2));
}

// 2. Get the rdb-mysql template
async function getMySQLTemplate() {
  const res = await fetch(`${BASE_URL}/ds-templates/rdb-mysql`, {
    method: "GET",
    headers: HEADERS
  });
  const data = await res.json();
  console.log("MySQL template:", JSON.stringify(data, null, 2));
}

// 3. Test the connection
async function testConnection() {
  const body = {
    ds_connection: {
      username: "bigid",
      password: "bigid111",
      rdb_url: "sql.mybigid.com",
      type: "rdb-mysql",
      enabled: "yes",
      rdb_name: "rockstream"
    },
    isNewPassword: true
  };

  const res = await fetch(`${BASE_URL}/ds-connection-test`, {
    method: "POST",
    headers: HEADERS,
    body: JSON.stringify(body)
  });

  const data = await res.json();
  console.log("Connection test result:", JSON.stringify(data, null, 2));
}

// 4. Add the data source
async function addMySQLDataSource() {
  const body = {
    ds_connection: {
      name: "645",
      type: "rdb-mysql",
      rdb_url: "sql.mybigid.com",
      username: "bigid",
      password: "bigid111",
      rdb_name: "rockstream"
    }
  };

  const res = await fetch(`${BASE_URL}/ds_connections`, {
    method: "POST",
    headers: HEADERS,
    body: JSON.stringify(body)
  });

  const data = await res.json();
  console.log("Add data source response:", JSON.stringify(data, null, 2));
}

async function main() {
  await getTemplates();
  await getMySQLTemplate();
  await testConnection();
  await addDataSource();
}

main().catch(err => {
  console.error("Error in tutorial flow:", err);
});