from scrapedatshi import ScrapedatshiClient

client = ScrapedatshiClient()

# chunk_file() parses + chunks a local PDF (all tiers)
result = client.pipeline.chunk_file("./report.pdf")
print(f"Got {result.total_chunks} chunks from {result.source}")

import os
import requests

API_KEY = os.getenv("SCRAPEDATSHI_API_KEY")
ENDPOINT = "https://www.scrapedatshi.com/api/pdf/text"
headers = {"X-API-Key": API_KEY}

# Option A: extract from a URL
data = requests.post(
    ENDPOINT,
    data={"url": "https://example.com/report.pdf", "preserve_headings": "true"},
    headers=headers
).json()

# Option B: upload a local file
with open("local_document.pdf", "rb") as f:
    data = requests.post(
        ENDPOINT,
        data={"preserve_headings": "false"},
        files={"pdf_file": f},
        headers=headers
    ).json()

print(data["text"])

from scrapedatshi import ScrapedatshiClient

client = ScrapedatshiClient()  # reads SCRAPEDATSHI_API_KEY from env

result = client.pipeline.chunk_url(
    "https://example.com/article",
    # Optional: contextual retrieval (Basic tier+)
    # contextual_retrieval=True,
    # llm_provider="openai",
    # llm_api_key=os.getenv("OPENAI_API_KEY"),
)

print(f"Chunks: {result.total_chunks}  |  Source: {result.source}")
for chunk in result.chunks:
    print(f"  [{chunk.token_estimate} tokens] {chunk.content[:100]}...")

import os
import requests

API_KEY = os.getenv("SCRAPEDATSHI_API_KEY")
ENDPOINT = "https://www.scrapedatshi.com/v1/rag-chunk"
headers = {"X-API-Key": API_KEY, "Content-Type": "application/json"}

payload = {
    "url": "https://example.com/article",
    "selector": "article",
    "chunk_size": 512,
    "overlap": 50,
}

response = requests.post(ENDPOINT, json=payload, headers=headers)
data = response.json()

print(f"Chunks: {data['chunk_count']}  |  Tokens: {data['total_tokens_estimated']}")
for chunk in data["chunks"]:
    print(f"  [{chunk['token_estimate']} tokens] {chunk['text'][:100]}...")

from scrapedatshi import ScrapedatshiClient

client = ScrapedatshiClient()

# Sitemap mode — reads sitemap.xml, structured and predictable
result = client.pipeline.crawl(
    "https://docs.example.com",
    max_pages=10,
)

print(f"Crawled {result.pages_crawled} pages → {result.total_chunks} chunks")
for chunk in result.chunks:
    print(f"  {chunk.content[:80]}...")

# Spider mode — follows links recursively, works on any site
# Use include_pattern to keep it focused on the right section
result = client.pipeline.crawl(
    "https://example.com",
    max_pages=20,
)

import os
import requests

API_KEY = os.getenv("SCRAPEDATSHI_API_KEY")
ENDPOINT = "https://www.scrapedatshi.com/v1/crawl"
headers = {"X-API-Key": API_KEY, "Content-Type": "application/json"}

# Sitemap mode — reads sitemap.xml, structured and predictable
payload = {
    "url": "https://docs.stripe.com/",
    "include_pattern": "/docs/",   # guardrail: only /docs/ pages
    "exclude_pattern": "/changelog/",
    "max_pages": 25,
    "selector": "article",
    "chunk": True,
    "chunk_size": 512,
    "overlap": 50,
}

response = requests.post(ENDPOINT, json=payload, headers=headers)
data = response.json()

print(f"Crawled: {data['pages_crawled']} pages  |  Tokens: ~{data['total_tokens_estimated']:,}")

all_chunks = []
for page in data["pages"]:
    if page["status"] == "ok":
        for chunk in page.get("chunks", []):
            all_chunks.append({"text": chunk["text"], "url": page["url"]})

print(f"Total chunks ready for embedding: {len(all_chunks)}")

# Spider mode — follows links recursively, works on any site without a sitemap
# Use the /v1/autorag endpoint for the full spider + embed + inject pipeline
payload = {
    "url": "https://example.com",
    "crawl_mode": "spider",        # follows links instead of reading sitemap
    "include_pattern": "/docs/",   # guardrail: keep it focused
    "max_pages": 20,
    "chunk": True,
}

response = requests.post(
    "https://www.scrapedatshi.com/v1/crawl",
    json=payload,
    headers=headers
)
data = response.json()
print(f"Spider crawled: {data['pages_crawled']} pages")

import os
from scrapedatshi import ScrapedatshiClient

client = ScrapedatshiClient()

result = client.pipeline.extract(
    url="https://example.com/product/widget-pro",
    schema={
        "title": "string — the product name",
        "price": "number — price in USD, no currency symbol",
        "in_stock": "boolean — whether the product is available",
        "description": "string — short product description, 1-2 sentences",
    },
    llm_provider="openai",
    llm_api_key=os.getenv("OPENAI_API_KEY"),
    llm_model="gpt-4o-mini",
)

print(result.extracted)
# → {"title": "Widget Pro", "price": 49.99, "in_stock": True, "description": "..."}
print(f"Cost: ${result.credits_used:.4f}")

import os
import requests

SCRAPEDATSHI_KEY = os.getenv("SCRAPEDATSHI_API_KEY")
OPENAI_KEY = os.getenv("OPENAI_API_KEY")

ENDPOINT = "https://www.scrapedatshi.com/v1/extract"
headers = {"X-API-Key": SCRAPEDATSHI_KEY, "Content-Type": "application/json"}

payload = {
    "url": "https://example.com/product/widget-pro",
    "schema": {
        "title": "string — the product name",
        "price": "number — price in USD, no currency symbol",
        "in_stock": "boolean — whether the product is available",
        "description": "string — short product description, 1-2 sentences",
        "rating": "number — average customer rating out of 5, or null if not shown"
    },
    "llm_provider": "openai",
    "llm_api_key": OPENAI_KEY,
    "llm_model": "gpt-4o-mini",  # required — check your provider's docs for available models
    "selector": "main",           # optional — focus on main content
}

response = requests.post(ENDPOINT, json=payload, headers=headers)
data = response.json()

product = data["extracted"]
print(f"Title:       {product['title']}")
print(f"Price:       ${product['price']}")
print(f"In Stock:    {product['in_stock']}")
print(f"Rating:      {product['rating']}/5")
print(f"Description: {product['description']}")

import os
from scrapedatshi import ScrapedatshiClient

client = ScrapedatshiClient()

result = client.pipeline.sync(
    url="https://docs.example.com/getting-started",
    embedding_provider="openai",
    embedding_api_key=os.getenv("OPENAI_API_KEY"),
    embedding_model="text-embedding-3-small",
    vector_db="pinecone",
    vector_db_config={
        "api_key": os.getenv("PINECONE_API_KEY"),
        "index_host": os.getenv("PINECONE_INDEX_HOST"),  # e.g. https://my-index-xyz.pinecone.io
    },
)

print(f"Upserted {result.vectors_upserted} vectors ({result.total_tokens} tokens)")
print(f"Embedding: {result.embedding_provider}  |  Vector DB: {result.vector_db_provider}")

import os
import requests

SCRAPEDATSHI_KEY = os.getenv("SCRAPEDATSHI_API_KEY")
OPENAI_KEY = os.getenv("OPENAI_API_KEY")
PINECONE_KEY = os.getenv("PINECONE_API_KEY")
PINECONE_HOST = os.getenv("PINECONE_INDEX_HOST")  # e.g. https://my-index-xyz.pinecone.io

ENDPOINT = "https://www.scrapedatshi.com/v1/sync"
headers = {"X-API-Key": SCRAPEDATSHI_KEY, "Content-Type": "application/json"}

payload = {
    "url": "https://docs.example.com/getting-started",
    "selector": "article",
    "chunk_size": 512,
    "overlap": 50,
    "max_chunks": 200,
    "embedding": {
        "provider": "openai",
        "api_key": OPENAI_KEY,
        "model": "text-embedding-3-small",  # optional
    },
    "vector_db": {
        "provider": "pinecone",
        "api_key": PINECONE_KEY,
        "index_host": PINECONE_HOST,
        "namespace": "docs",  # optional
    },
}

response = requests.post(ENDPOINT, json=payload, headers=headers)
data = response.json()

print(f"Chunks created:    {data['chunks_created']}")
print(f"Vectors upserted:  {data['vectors_upserted']}")
print(f"Tokens estimated:  ~{data['total_tokens_estimated']:,}")
print(f"Embedding model:   {data['embedding_model']}")
print(f"Vector DB:         {data['vector_db_provider']}")

import os
from scrapedatshi import ScrapedatshiClient

client = ScrapedatshiClient()

# AutoRAG — crawl an entire site and inject into your vector DB
result = client.pipeline.autorag(
    url="https://docs.example.com",
    crawl_mode="sitemap",          # or "spider" for sites without a sitemap
    max_pages=500,                 # large jobs are auto-batched server-side
    include_pattern="/docs/",      # guardrail: only /docs/ pages
    embedding_provider="openai",
    embedding_api_key=os.getenv("OPENAI_API_KEY"),
    embedding_model="text-embedding-3-small",
    vector_db="pinecone",
    vector_db_config={
        "api_key": os.getenv("PINECONE_API_KEY"),
        "index_host": os.getenv("PINECONE_INDEX_HOST"),
    },
)

print(f"Crawled {result.pages_crawled} pages → {result.vectors_upserted} vectors")
if result.auto_batched:
    print(f"Auto-batched: {result.batches_processed} batches of {result.batch_size} pages")

import os
import requests

SCRAPEDATSHI_KEY = os.getenv("SCRAPEDATSHI_API_KEY")
OPENAI_KEY = os.getenv("OPENAI_API_KEY")
PINECONE_KEY = os.getenv("PINECONE_API_KEY")
PINECONE_HOST = os.getenv("PINECONE_INDEX_HOST")

ENDPOINT = "https://www.scrapedatshi.com/v1/autorag"
headers = {"X-API-Key": SCRAPEDATSHI_KEY, "Content-Type": "application/json"}

payload = {
    "url": "https://docs.example.com",
    "crawl_mode": "sitemap",       # or "spider"
    "max_pages": 500,              # large jobs auto-batch server-side
    "include_pattern": "/docs/",   # guardrail
    "chunk_size": 512,
    "overlap": 50,
    "embedding": {
        "provider": "openai",
        "api_key": OPENAI_KEY,
        "model": "text-embedding-3-small",
    },
    "vector_db": {
        "provider": "pinecone",
        "api_key": PINECONE_KEY,
        "index_host": PINECONE_HOST,
        "namespace": "docs",
    },
}

response = requests.post(ENDPOINT, json=payload, headers=headers)
data = response.json()

print(f"Pages crawled:    {data['pages_crawled']}")
print(f"Vectors upserted: {data['vectors_upserted']}")
print(f"Credits used:     ${data['credits_used']:.4f}")
if data.get("auto_batched"):
    print(f"Auto-batched:     {data['batches_processed']} × {data['batch_size']} pages")

# PDF Tables is a raw REST endpoint — use requests directly.
# See the Raw REST tab for the full example.

# To chunk a PDF file with the SDK instead:
from scrapedatshi import ScrapedatshiClient
client = ScrapedatshiClient()
result = client.pipeline.chunk_file("./report.pdf")
print(f"Got {result.total_chunks} chunks")

import os
import requests
import pandas as pd  # optional: load into a DataFrame

API_KEY = os.getenv("SCRAPEDATSHI_API_KEY")
ENDPOINT = "https://www.scrapedatshi.com/api/pdf/tables"
headers = {"X-API-Key": API_KEY}

response = requests.post(
    ENDPOINT,
    data={"url": "https://example.com/report.pdf"},
    headers=headers
)
data = response.json()

print(f"Found {data['table_count']} table(s)")

for table in data["tables"]:
    print(f"\nTable {table['table_index']} (page {table['page']}):")
    df = pd.DataFrame(table["rows"][1:], columns=table["rows"][0])
    print(df.to_string(index=False))