AI Data Profiling

curl -s -X POST http://localhost:8080/api/v1/pipeline/profile \
  -F "file=@stock_prices.csv" \
  -F "delimiter=," \
  -F "header=true" \
  -F "sampleSize=200" | python3 -m json.tool

{
  "summary": {
    "rowCount": 200,
    "columnCount": 8,
    "columns": [
      {
        "name": "symbol",
        "inferredType": "string",
        "nullCount": 0,
        "uniqueCount": 195,
        "sampleValues": ["FAX", "IAF", "FCO"]
      },
      {
        "name": "date",
        "inferredType": "date",
        "nullCount": 0,
        "uniqueCount": 1,
        "sampleValues": ["2016-12-30"]
      },
      {
        "name": "open",
        "inferredType": "float",
        "nullCount": 1,
        "uniqueCount": 198,
        "sampleValues": ["4.65", "5.44", "7.91"]
      }
    ]
  },
  "qualityIssues": [
    "Column 'open' has 1 null/empty value",
    "Column 'volume' contains a negative value (-500)",
    "Column 'close' has a value exceeding $1,000,000"
  ],
  "recommendations": [
    "Add an aiRule to validate symbol format, price ranges, and cross-column relationships",
    "Consider sampling for large files to keep processing fast"
  ],
  "suggestedDataQuality": {
    "aiRule": {
      "instruction": "symbol must be 1-5 uppercase letters, all price columns (open, high, low, close, adj_close) must be positive and not exceed $1,000,000, volume must be positive, and high must be greater than or equal to low",
      "onFailureIsError": false
    }
  }
}