Wait when rendering Javascript

Let's say you want to scrape a website that renders its data slowly then in that case, you can use our wait parameter along with the dynamic parameter. wait parameter is actually a time for which you want to wait for a headless browser to keep loading the website. It is in milliseconds and its minimum value is 0 and the maximum value is 35000.

API Example

curl "https://api.scrapingdog.com/scrape?api_key=5e5a97e5b1ca5b194f42da86&url=http://httpbin.org/ip&wait=5000"

import requests

url = "https://api.scrapingdog.com/scrape"

params = {
    "api_key": "5e5a97e5b1ca5b194f42da86",
    "url": "http://httpbin.org/ip",
    "wait": "5000"
}

response = requests.get(url, params=params)

print(response.text)

const axios = require('axios');

const apiUrl = 'https://api.scrapingdog.com/scrape';
const apiKey = '5e5a97e5b1ca5b194f42da86';
const targetUrl = 'http://httpbin.org/ip';

const params = {
  api_key: apiKey,
  url: targetUrl,
  wait: "5000"
  dynamic: dynamic.toString(),
};

axios
  .get(apiUrl, { params })
  .then((response) => {
    if (response.status === 200) {
      console.log(response.data);
    } else {
      console.error(`Failed to retrieve data. Status code: ${response.status}`);
    }
  })
  .catch((error) => {
    console.error('An error occurred:', error.message);
  });

<?php
$apiUrl = 'https://api.scrapingdog.com/scrape';
$apiKey = '5e5a97e5b1ca5b194f42da86';
$targetUrl = 'http://httpbin.org/ip';
$waitingTime = '5000';

$queryParams = [
    'api_key' => $apiKey,
    'url' => $targetUrl,
    'dynamic' => $dynamic,
    'wait' => $waitingTime,
];

$queryString = http_build_query($queryParams);

$fullUrl = $apiUrl . '?' . $queryString;

$curl = curl_init();
curl_setopt($curl, CURLOPT_URL, $fullUrl);
curl_setopt($curl, CURLOPT_RETURNTRANSFER, true);

$response = curl_exec($curl);

if ($response === false) {
    echo 'cURL error: ' . curl_error($curl);
} else {
    $httpCode = curl_getinfo($curl, CURLINFO_HTTP_CODE);
    if ($httpCode === 200) {
        echo $response;
    } else {
        echo 'Failed to retrieve data. Status code: ' . $httpCode;
    }
}

curl_close($curl);
?>

require 'net/http'
require 'uri'

api_url = 'https://api.scrapingdog.com/scrape'
api_key = '5e5a97e5b1ca5b194f42da86'
target_url = 'http://httpbin.org/ip'
waitingTime = '5000'

uri = URI.parse(api_url)
params = {
  'api_key' => api_key,
  'url' => target_url,
  'dynamic' => dynamic,
  'wait' => waitingTime,
}

uri.query = URI.encode_www_form(params)

http = Net::HTTP.new(uri.host, uri.port)
http.use_ssl = true

request = Net::HTTP::Get.new(uri.request_uri)

response = http.request(request)

if response.code == '200'
  puts response.body
else
  puts "Failed to retrieve data. Status code: #{response.code}"
end

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;

public class ScrapingDogAPITest {
    public static void main(String[] args) {
        try {
            String apiUrl = "https://api.scrapingdog.com/scrape";
            String apiKey = "5e5a97e5b1ca5b194f42da86";
            String targetUrl = "http://httpbin.org/ip";
            String waitingTime = "5000";


            // Construct the query parameters
            Map<String, String> params = new HashMap<>();
            params.put("api_key", apiKey);
            params.put("url", targetUrl);
            params.put("wait", waitingTime);

            // Build the query URL
            StringBuilder query = new StringBuilder(apiUrl);
            query.append("?");
            for (Map.Entry<String, String> entry : params.entrySet()) {
                query.append(entry.getKey()).append("=").append(entry.getValue()).append("&");
            }
            String queryUrl = query.toString().substring(0, query.length() - 1);

            // Create an HTTP connection and set up the request
            URL url = new URL(queryUrl);
            HttpURLConnection connection = (HttpURLConnection) url.openConnection();
            connection.setRequestMethod("GET");

            // Get the response code
            int responseCode = connection.getResponseCode();

            if (responseCode == HttpURLConnection.HTTP_OK) {
                // Read and print the response
                BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream()));
                String inputLine;
                StringBuilder response = new StringBuilder();

                while ((inputLine = in.readLine()) != null) {
                    response.append(inputLine);
                }

                in.close();
                System.out.println(response.toString());
            } else {
                System.out.println("Failed to retrieve data. Status code: " + responseCode);
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}

PreviousJavascript Rendering NextCustom Headers

Last updated 2 years ago

import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import java.net.HttpURLConnection; import java.net.URL; import java.util.HashMap; import java.util.Map; public class ScrapingDogAPITest { public static void main(String[] args) { try { String apiUrl = "https://api.scrapingdog.com/scrape"; String apiKey = "5e5a97e5b1ca5b194f42da86"; String targetUrl = "http://httpbin.org/ip"; String waitingTime = "5000"; // Construct the query parameters Map<String, String> params = new HashMap<>(); params.put("api_key", apiKey); params.put("url", targetUrl); params.put("wait", waitingTime); // Build the query URL StringBuilder query = new StringBuilder(apiUrl); query.append("?"); for (Map.Entry<String, String> entry : params.entrySet()) { query.append(entry.getKey()).append("=").append(entry.getValue()).append("&"); } String queryUrl = query.toString().substring(0, query.length() - 1); // Create an HTTP connection and set up the request URL url = new URL(queryUrl); HttpURLConnection connection = (HttpURLConnection) url.openConnection(); connection.setRequestMethod("GET"); // Get the response code int responseCode = connection.getResponseCode(); if (responseCode == HttpURLConnection.HTTP_OK) { // Read and print the response BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream())); String inputLine; StringBuilder response = new StringBuilder(); while ((inputLine = in.readLine()) != null) { response.append(inputLine); } in.close(); System.out.println(response.toString()); } else { System.out.println("Failed to retrieve data. Status code: " + responseCode); } } catch (IOException e) { e.printStackTrace(); } } }