Comment on page
Web Scraping API
With a simple GET request to our Web Scraping API, you can scrape any webpage.
Simply send a GET request to
https://api.scrapingdog.com/scrape with two query string parameters, api_key which is your personal API key, and url which is the target URL you would like to scrape.
Parameter | Description |
|---|---|
api_key
required | Your personal API key. Available on your dashboard
Type: String |
url
required | URL of the page you want to scrape.
Type: String |
dynamic | It tells our server whether you want to render JS or not. It can be either true or false. By default it is true.
Type: Boolean |
Curl
Python
Nodejs
PHP
Ruby
Java
curl "https://api.scrapingdog.com/scrape?api_key=5e5a97e5b1ca5b194f42da86&url=http://httpbin.org/ip&dynamic=false"
import requests
url = "https://api.scrapingdog.com/scrape"
params = {
"api_key": "5e5a97e5b1ca5b194f42da86",
"url": "http://httpbin.org/ip",
"dynamic": "false"
}
response = requests.get(url, params=params)
print(response.text)
const axios = require('axios');
const apiUrl = 'https://api.scrapingdog.com/scrape';
const apiKey = '5e5a97e5b1ca5b194f42da86';
const targetUrl = 'http://httpbin.org/ip';
const dynamic = false;
const params = {
api_key: apiKey,
url: targetUrl,
dynamic: dynamic.toString(),
};
axios
.get(apiUrl, { params })
.then((response) => {
if (response.status === 200) {
console.log(response.data);
} else {
console.error(`Failed to retrieve data. Status code: ${response.status}`);
}
})
.catch((error) => {
console.error('An error occurred:', error.message);
});
<?php
$apiUrl = 'https://api.scrapingdog.com/scrape';
$apiKey = '5e5a97e5b1ca5b194f42da86';
$targetUrl = 'http://httpbin.org/ip';
$dynamic = false;
$queryParams = [
'api_key' => $apiKey,
'url' => $targetUrl,
'dynamic' => $dynamic,
];
$queryString = http_build_query($queryParams);
$fullUrl = $apiUrl . '?' . $queryString;
$curl = curl_init();
curl_setopt($curl, CURLOPT_URL, $fullUrl);
curl_setopt($curl, CURLOPT_RETURNTRANSFER, true);
$response = curl_exec($curl);
if ($response === false) {
echo 'cURL error: ' . curl_error($curl);
} else {
$httpCode = curl_getinfo($curl, CURLINFO_HTTP_CODE);
if ($httpCode === 200) {
echo $response;
} else {
echo 'Failed to retrieve data. Status code: ' . $httpCode;
}
}
curl_close($curl);
?>
require 'net/http'
require 'uri'
api_url = 'https://api.scrapingdog.com/scrape'
api_key = '5e5a97e5b1ca5b194f42da86'
target_url = 'http://httpbin.org/ip'
dynamic = false
uri = URI.parse(api_url)
params = {
'api_key' => api_key,
'url' => target_url,
'dynamic' => dynamic
}
uri.query = URI.encode_www_form(params)
http = Net::HTTP.new(uri.host, uri.port)
http.use_ssl = true
request = Net::HTTP::Get.new(uri.request_uri)
response = http.request(request)
if response.code == '200'
puts response.body
else
puts "Failed to retrieve data. Status code: #{response.code}"
end
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
public class ScrapingDogAPITest {
public static void main(String[] args) {
try {
String apiUrl = "https://api.scrapingdog.com/scrape";
String apiKey = "5e5a97e5b1ca5b194f42da86";
String targetUrl = "http://httpbin.org/ip";
boolean dynamic = false;
// Construct the query parameters
Map<String, String> params = new HashMap<>();
params.put("api_key", apiKey);
params.put("url", targetUrl);
params.put("dynamic", String.valueOf(dynamic));
// Build the query URL
StringBuilder query = new StringBuilder(apiUrl);
query.append("?");
for (Map.Entry<String, String> entry : params.entrySet()) {
query.append(entry.getKey()).append("=").append(entry.getValue()).append("&");
}
String queryUrl = query.toString().substring(0, query.length() - 1);
// Create an HTTP connection and set up the request
URL url = new URL(queryUrl);
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setRequestMethod("GET");
// Get the response code
int responseCode = connection.getResponseCode();
if (responseCode == HttpURLConnection.HTTP_OK) {
// Read and print the response
BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream()));
String inputLine;
StringBuilder response = new StringBuilder();
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
in.close();
System.out.println(response.toString());
} else {
System.out.println("Failed to retrieve data. Status code: " + responseCode);
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
<html>
<head>
</head>
<body>
<pre style="word-wrap: break-word; white-space: pre-wrap;">
{"origin":"27.63.83.45"}
</pre>
</body>
</html>
Last modified 1mo ago