Web Scraping API

With a simple GET request to our Web Scraping API, you can scrape any webpage.

Simply send a GET request to https://api.scrapingdog.com/scrape with two query string parameters, api_key which is your personal API key, and url which is the target URL you would like to scrape.

Parameters

Parameter

Description

api_key required

Your personal API key. Available on your dashboard Type: String

url required

URL of the page you want to scrape. Type: String

dynamic

It tells our server whether you want to render JS or not. It can be either true or false. By default it is true. Type: Boolean

Usage

curl "https://api.scrapingdog.com/scrape?api_key=5e5a97e5b1ca5b194f42da86&url=http://httpbin.org/ip&dynamic=false"

import requests

url = "https://api.scrapingdog.com/scrape"

params = {
    "api_key": "5e5a97e5b1ca5b194f42da86",
    "url": "http://httpbin.org/ip",
    "dynamic": "false"
}

response = requests.get(url, params=params)

print(response.text)

const axios = require('axios');

const apiUrl = 'https://api.scrapingdog.com/scrape';
const apiKey = '5e5a97e5b1ca5b194f42da86';
const targetUrl = 'http://httpbin.org/ip';
const dynamic = false;

const params = {
  api_key: apiKey,
  url: targetUrl,
  dynamic: dynamic.toString(),
};

axios
  .get(apiUrl, { params })
  .then((response) => {
    if (response.status === 200) {
      console.log(response.data);
    } else {
      console.error(`Failed to retrieve data. Status code: ${response.status}`);
    }
  })
  .catch((error) => {
    console.error('An error occurred:', error.message);
  });

<?php
$apiUrl = 'https://api.scrapingdog.com/scrape';
$apiKey = '5e5a97e5b1ca5b194f42da86';
$targetUrl = 'http://httpbin.org/ip';
$dynamic = false;

$queryParams = [
    'api_key' => $apiKey,
    'url' => $targetUrl,
    'dynamic' => $dynamic,
];

$queryString = http_build_query($queryParams);

$fullUrl = $apiUrl . '?' . $queryString;

$curl = curl_init();
curl_setopt($curl, CURLOPT_URL, $fullUrl);
curl_setopt($curl, CURLOPT_RETURNTRANSFER, true);

$response = curl_exec($curl);

if ($response === false) {
    echo 'cURL error: ' . curl_error($curl);
} else {
    $httpCode = curl_getinfo($curl, CURLINFO_HTTP_CODE);
    if ($httpCode === 200) {
        echo $response;
    } else {
        echo 'Failed to retrieve data. Status code: ' . $httpCode;
    }
}

curl_close($curl);
?>

require 'net/http'
require 'uri'

api_url = 'https://api.scrapingdog.com/scrape'
api_key = '5e5a97e5b1ca5b194f42da86'
target_url = 'http://httpbin.org/ip'
dynamic = false

uri = URI.parse(api_url)
params = {
  'api_key' => api_key,
  'url' => target_url,
  'dynamic' => dynamic
}

uri.query = URI.encode_www_form(params)

http = Net::HTTP.new(uri.host, uri.port)
http.use_ssl = true

request = Net::HTTP::Get.new(uri.request_uri)

response = http.request(request)

if response.code == '200'
  puts response.body
else
  puts "Failed to retrieve data. Status code: #{response.code}"
end

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;

public class ScrapingDogAPITest {
    public static void main(String[] args) {
        try {
            String apiUrl = "https://api.scrapingdog.com/scrape";
            String apiKey = "5e5a97e5b1ca5b194f42da86";
            String targetUrl = "http://httpbin.org/ip";
            boolean dynamic = false;

            // Construct the query parameters
            Map<String, String> params = new HashMap<>();
            params.put("api_key", apiKey);
            params.put("url", targetUrl);
            params.put("dynamic", String.valueOf(dynamic));

            // Build the query URL
            StringBuilder query = new StringBuilder(apiUrl);
            query.append("?");
            for (Map.Entry<String, String> entry : params.entrySet()) {
                query.append(entry.getKey()).append("=").append(entry.getValue()).append("&");
            }
            String queryUrl = query.toString().substring(0, query.length() - 1);

            // Create an HTTP connection and set up the request
            URL url = new URL(queryUrl);
            HttpURLConnection connection = (HttpURLConnection) url.openConnection();
            connection.setRequestMethod("GET");

            // Get the response code
            int responseCode = connection.getResponseCode();

            if (responseCode == HttpURLConnection.HTTP_OK) {
                // Read and print the response
                BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream()));
                String inputLine;
                StringBuilder response = new StringBuilder();

                while ((inputLine = in.readLine()) != null) {
                    response.append(inputLine);
                }

                in.close();
                System.out.println(response.toString());
            } else {
                System.out.println("Failed to retrieve data. Status code: " + responseCode);
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}

Response

<html>
  <head>
  </head>
  <body>
    <pre style="word-wrap: break-word; white-space: pre-wrap;">
      {"origin":"27.63.83.45"}
    </pre>
  </body>
</html>

PreviousDocumentation NextRequest Customization

Last updated 1 year ago

import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import java.net.HttpURLConnection; import java.net.URL; import java.util.HashMap; import java.util.Map; public class ScrapingDogAPITest { public static void main(String[] args) { try { String apiUrl = "https://api.scrapingdog.com/scrape"; String apiKey = "5e5a97e5b1ca5b194f42da86"; String targetUrl = "http://httpbin.org/ip"; boolean dynamic = false; // Construct the query parameters Map<String, String> params = new HashMap<>(); params.put("api_key", apiKey); params.put("url", targetUrl); params.put("dynamic", String.valueOf(dynamic)); // Build the query URL StringBuilder query = new StringBuilder(apiUrl); query.append("?"); for (Map.Entry<String, String> entry : params.entrySet()) { query.append(entry.getKey()).append("=").append(entry.getValue()).append("&"); } String queryUrl = query.toString().substring(0, query.length() - 1); // Create an HTTP connection and set up the request URL url = new URL(queryUrl); HttpURLConnection connection = (HttpURLConnection) url.openConnection(); connection.setRequestMethod("GET"); // Get the response code int responseCode = connection.getResponseCode(); if (responseCode == HttpURLConnection.HTTP_OK) { // Read and print the response BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream())); String inputLine; StringBuilder response = new StringBuilder(); while ((inputLine = in.readLine()) != null) { response.append(inputLine); } in.close(); System.out.println(response.toString()); } else { System.out.println("Failed to retrieve data. Status code: " + responseCode); } } catch (IOException e) { e.printStackTrace(); } } }