Web scraping Google Trends with Nodejs
Full code
If you don’t need an explanation, have a look at the full code example in the online IDE
const puppeteer = require("puppeteer-extra");
const StealthPlugin = require("puppeteer-extra-plugin-stealth");puppeteer.use(StealthPlugin());const searchQueries = ["Mercedes"]; // what we want to search (for interestOverTime, interestByRegion, relatedQueries, relatedTopics)
// const searchQueries = ["Mercedes", "BMW", "Audi"]; // what we want to search (for interestOverTime, comparedByRegion, interestByRegion, relatedQueries)const URL = `https://trends.google.com/trends/explore?q=${encodeURI(searchQueries.join(","))}&hl=en`;async function getGoogleTrendsResults() {
const browser = await puppeteer.launch({
headless: false,
args: ["--no-sandbox", "--disable-setuid-sandbox"],
});
const page = await browser.newPage();
await page.setDefaultNavigationTimeout(60000);
await page.goto(URL);
await page.waitForTimeout(5000);
await page.reload();
const interestOverTime = {};
const comparedByRegion = [];
const interestByRegion = [];
const relatedQueries = [];
const relatedTopics = {}; const valuePattern = /%22value%22:%22(?<value>[^%]+)/gm; //https://regex101.com/r/PNcP1u/1 page.on("response", async (response) => {
if (response.headers()["content-type"]?.includes("application/")) {
const responseData = await response.text();
const responseURL = await response.url();
if (responseURL.includes("widgetdata/multiline?")) {
const parsedData = JSON.parse(responseData.slice(6))?.default;
interestOverTime.timelineData = parsedData?.timelineData?.map((dataEl) => ({
date: decodeURI(dataEl.formattedTime),
values: searchQueries.map((queryEl, i) => ({
query: queryEl,
value: dataEl.formattedValue[i],
extractedValue: dataEl.value[i],
})),
}));
interestOverTime.averages = parsedData.averages.map((dataEl, i) => ({
query: searchQueries[i],
value: dataEl,
}));
} else {
const values = [...responseURL.matchAll(valuePattern)].map(({ groups }) => groups.value);
if (responseURL.includes("widgetdata/comparedgeo?")) {
if (values.length > 1) {
const parsedData = JSON.parse(responseData.slice(6))?.default;
comparedByRegion.push(
...parsedData.geoMapData.map((dataEl) => ({
geo: dataEl.geoCode,
location: dataEl.geoName,
maxValueIndex: dataEl.maxValueIndex,
values: searchQueries.map((queryEl, i) => ({
query: queryEl,
value: dataEl.formattedValue[i],
extractedValue: dataEl.value[i],
})),
}))
);
} else {
for (query of searchQueries) {
if (values[0] === query) {
const parsedData = JSON.parse(responseData.slice(6))?.default;
interestByRegion.push({
query,
data: parsedData.geoMapData.map((dataEl) => ({
geo: dataEl.geoCode,
location: dataEl.geoName,
maxValueIndex: dataEl.maxValueIndex,
value: dataEl.formattedValue[0],
extractedValue: dataEl.value[0],
})),
});
}
}
}
} else if (responseURL.includes("widgetdata/relatedsearches?")) {
for (query of searchQueries) {
if (values[0] === query) {
if (responseURL.includes("%22keywordType%22:%22ENTITY%22")) {
const parsedData = JSON.parse(responseData.slice(6))?.default;
relatedTopics.top = parsedData.rankedList[0].rankedKeyword.map((dataEl) => ({
topic: {
title: dataEl.topic.title,
type: dataEl.topic.type,
},
value: dataEl.formattedValue,
extractedValue: dataEl.value,
link: "https://trends.google.com" + dataEl.link,
}));
relatedTopics.rising = parsedData.rankedList[1].rankedKeyword.map((dataEl) => ({
topic: {
title: dataEl.topic.title,
type: dataEl.topic.type,
},
value: dataEl.formattedValue,
extractedValue: dataEl.value,
link: "https://trends.google.com" + dataEl.link,
}));
} else {
const parsedData = JSON.parse(responseData.slice(6))?.default;
relatedQueries.push({
searchQuery: query,
top: parsedData.rankedList[0].rankedKeyword.map((dataEl) => ({
query: dataEl.query,
value: dataEl.formattedValue,
extractedValue: dataEl.value,
link: "https://trends.google.com" + dataEl.link,
})),
rising: parsedData.rankedList[1].rankedKeyword.map((dataEl) => ({
query: dataEl.query,
value: dataEl.formattedValue,
extractedValue: dataEl.value,
link: "https://trends.google.com" + dataEl.link,
})),
});
}
}
}
}
}
}
}); await page.waitForTimeout(10000);
await browser.close(); return { interestOverTime, comparedByRegion, interestByRegion, relatedQueries, relatedTopics };
}getGoogleTrendsResults().then((result) => console.dir(result, { depth: null }));
Preparation
First, we need to create a Node.js* project and add npm
packages puppeteer
, puppeteer-extra
and puppeteer-extra-plugin-stealth
to control Chromium (or Chrome, or Firefox, but now we work only with Chromium which is used by default) over the DevTools Protocol in headless or non-headless mode.
To do this, in the directory with our project, open the command line and enter npm init -y
, and then npm i puppeteer puppeteer-extra puppeteer-extra-plugin-stealth
.
*If you don’t have Node.js installed, you can download it from nodejs.org and follow the installation documentation.
📌Note: also, you can use puppeteer
without any extensions, but I strongly recommended use it with puppeteer-extra
with puppeteer-extra-plugin-stealth
to prevent website detection that you are using headless Chromium or that you are using web driver. You can check it on Chrome headless tests website. The screenshot below shows you a difference.
Process
In this case, we need to intercept all responses when loading the page and get all the required data. The Gif below illustrates that approach:
Code explanation
Declare puppeteer
to control Chromium browser from puppeteer-extra
library and StealthPlugin
to prevent website detection that you are using web driver from puppeteer-extra-plugin-stealth
library:
const puppeteer = require("puppeteer-extra");
const StealthPlugin = require("puppeteer-extra-plugin-stealth");
Next, we “say” to puppeteer
use StealthPlugin
, write search queries and the search URL. I write two different search queries to show you all available results (some of them are available only for a single search query, but some are available only for multiple search queries):
const searchQueries = ["Mercedes"];
// const searchQueries = ["Mercedes", "BMW", "Audi"];const URL = `https://trends.google.com/trends/explore?q=${encodeURI(searchQueries.join(","))}&hl=en`;
Next, write a function to control the browser, and get information:
async function getGoogleTrendsResults() {
...
}
In this function first we need to define browser
using puppeteer.launch({options})
method with current options
, such as headless: false
and args: ["--no-sandbox", "--disable-setuid-sandbox"]
.
These options mean that we use headless mode and array with arguments which we use to allow the launch of the browser process in the online IDE. And then we open a new page
:
const browser = await puppeteer.launch({
headless: false,
args: ["--no-sandbox", "--disable-setuid-sandbox"],
});const page = await browser.newPage();
Next, we change default (30 sec) time for waiting for selectors to 60000 ms (1 min) for slow internet connection with .setDefaultNavigationTimeout()
method, go to URL
with .goto()
method and use .waitForTimeout()
method to wait 5 seconds:
await page.setDefaultNavigationTimeout(60000);
await page.goto(URL);
await page.waitForTimeout(5000);
Because google possibly blocks direct search requests we need to reload the page:
await page.reload();
Then, we define arrays and objects with the results and write valuePattern
to find the correct responses from all of them:
const interestOverTime = {};
const comparedByRegion = [];
const interestByRegion = [];
const relatedQueries = [];
const relatedTopics = {};const valuePattern = /%22value%22:%22(?<value>[^%]+)/gm;
Next, we right response intercept function:
page.on("response", async (response) => {
...
})
In this function first, we check that the response header content-type
have the "application/"
part. And if its true
we define and write responseData
with all data comes from current response and responseUrl
with URL:
if (response.headers()["content-type"]?.includes("application/")) {
const responseData = await response.text();
const responseURL = await response.url();
...
}
Next, we need to separate response with interestOverTime
data from all other:
if (responseURL.includes("widgetdata/multiline?")) {
...
} else {
...
}
If response with interestOverTime
data, we need to get the correct data to work with it. To do this we remove the first six chars ()]}',
and the new line /n
) from responseData
(.slice(6)
method), parse received JSON string to JS object (JSON.parse()
method) and get data from default
key:
const parsedData = JSON.parse(responseData.slice(6))?.default;
Next, we build the timelineData
array from our parsed response data with objects containing date
and values
keys and the averages
array with objects containing query
and value
keys (to get the correct date
we need to decode the URI encoded string):
interestOverTime.timelineData = parsedData?.timelineData?.map((dataEl) => ({
date: decodeURI(dataEl.formattedTime),
values: searchQueries.map((queryEl, i) => ({
query: queryEl,
value: dataEl.formattedValue[i],
extractedValue: dataEl.value[i],
})),
}));
interestOverTime.averages = parsedData.averages.map((dataEl, i) => ({
query: searchQueries[i],
value: dataEl,
}));
Next, to get all other data we need to get values
from responseURL
using the matchAll
method and the spread syntax([...]
):
const values = [...responseURL.matchAll(valuePattern)].map(({ groups }) => groups.value);
Next, to get comparedByRegion
data we need a response in which values
array length is more than one:
if (values.length > 1) {
const parsedData = JSON.parse(responseData.slice(6))?.default;
comparedByRegion.push(
...parsedData.geoMapData.map((dataEl) => ({
geo: dataEl.geoCode,
location: dataEl.geoName,
maxValueIndex: dataEl.maxValueIndex,
values: searchQueries.map((queryEl, i) => ({
query: queryEl,
value: dataEl.formattedValue[i],
extractedValue: dataEl.value[i],
})),
}))
);
}
Otherwise, we can get interestByRegion
data. To do this, we need to iterate searchQueries
array with the for...of
loop. In the loop, we compare the single value values
with query
. If it's equal to each other, we can fill the interestByRegion
array by pushing results from parsedData
:
else {
for (query of searchQueries) {
if (values[0] === query) {
const parsedData = JSON.parse(responseData.slice(6))?.default;
interestByRegion.push({
query,
data: parsedData.geoMapData.map((dataEl) => ({
geo: dataEl.geoCode,
location: dataEl.geoName,
maxValueIndex: dataEl.maxValueIndex,
value: dataEl.formattedValue[0],
extractedValue: dataEl.value[0],
})),
});
}
}
}
Next, to get relatedTopics
and relatedQueries
data we need a response with ""widgetdata/relatedsearches?""
part in the responseURL
. If it's true, we need to iterate over searchQueries
and compare the single value values
with query
again:
for (query of searchQueries) {
if (values[0] === query) {
...
}
}
Next, we check if responseURL
contains "%22keywordType%22:%22ENTITY%22"
part, we can get relatedTopics
data:
if (responseURL.includes("%22keywordType%22:%22ENTITY%22")) {
const parsedData = JSON.parse(responseData.slice(6))?.default;
relatedTopics.top = parsedData.rankedList[0].rankedKeyword.map((dataEl) => ({
topic: {
title: dataEl.topic.title,
type: dataEl.topic.type,
},
value: dataEl.formattedValue,
extractedValue: dataEl.value,
link: "https://trends.google.com" + dataEl.link,
}));
relatedTopics.rising = parsedData.rankedList[1].rankedKeyword.map((dataEl) => ({
topic: {
title: dataEl.topic.title,
type: dataEl.topic.type,
},
value: dataEl.formattedValue,
extractedValue: dataEl.value,
link: "https://trends.google.com" + dataEl.link,
}));
}
Otherwise, we can get relatedQueries
data:
else {
const parsedData = JSON.parse(responseData.slice(6))?.default;
relatedQueries.push({
searchQuery: query,
top: parsedData.rankedList[0].rankedKeyword.map((dataEl) => ({
query: dataEl.query,
value: dataEl.formattedValue,
extractedValue: dataEl.value,
link: "https://trends.google.com" + dataEl.link,
})),
rising: parsedData.rankedList[1].rankedKeyword.map((dataEl) => ({
query: dataEl.query,
value: dataEl.formattedValue,
extractedValue: dataEl.value,
link: "https://trends.google.com" + dataEl.link,
})),
});
}
And finally, we wait 10 seconds (wait for all responses to be finished) and close the browser and return the received data:
await page.waitForTimeout(10000);
await browser.close();return { interestOverTime, comparedByRegion, interestByRegion, relatedQueries, relatedTopics };
Now we can launch our parser:
$ node YOUR_FILE_NAME # YOUR_FILE_NAME is the name of your .js file
Output
📌Note: I’ve combined the results of different runs of our parser into one for convenience.
{
"interestOverTime":{
"timelineData":[
{
"date":"Aug 29 – Sep 4, 2021",
"values":[
{
"query":"Mercedes",
"value":"74",
"extractedValue":74
},
{
"query":"BMW",
"value":"100",
"extractedValue":100
},
{
"query":"Audi",
"value":"60",
"extractedValue":60
}
]
},
... and other dates
],
"averages":[
{
"query":"Mercedes",
"value":65
},
{
"query":"BMW",
"value":89
},
{
"query":"Audi",
"value":55
}
]
},
"comparedByRegion":[
{
"geo":"LT",
"location":"Lithuania",
"maxValueIndex":1,
"values":[
{
"query":"Mercedes",
"value":"13%",
"extractedValue":13
},
{
"query":"BMW",
"value":"52%",
"extractedValue":52
},
{
"query":"Audi",
"value":"35%",
"extractedValue":35
}
]
},
... and other regions
],
"interestByRegion": {
"query":"Mercedes",
"data":[
{
"geo":"GR",
"location":"Greece",
"maxValueIndex":0,
"value":"36",
"extractedValue":36
},
... and other regions
],
"query":"BMW",
"data":[
{
"geo":"GR",
"location":"Greece",
"maxValueIndex":0,
"value":"37",
"extractedValue":37
},
... and other regions
],
"query":"Audi",
"data":[
{
"geo":"XK",
"location":"Kosovo",
"maxValueIndex":0,
"value":"100",
"extractedValue":100
},
... and other regions
]
},
"relatedQueries":[
{
"searchQuery":"Audi",
"top":[
{
"query":"a4 audi",
"value":"100",
"extractedValue":100,
"link":"https://trends.google.com/trends/explore?q=a4+audi&date=today+12-m"
},
... and other queries
],
"rising":[
{
"query":"wheel of fortune contestant loses audi",
"value":"+2,800%",
"extractedValue":2800,
"link":"https://trends.google.com/trends/explore?q=wheel+of+fortune+contestant+loses+audi&date=today+12-m"
},
... and other queries
]
},
{
"searchQuery":"BMW",
"top":[
{
"query":"bmw price",
"value":"100",
"extractedValue":100,
"link":"https://trends.google.com/trends/explore?q=bmw+price&date=today+12-m"
},
... and other queries
],
"rising":[
{
"query":"bmw x6 2022",
"value":"+1,450%",
"extractedValue":1450,
"link":"https://trends.google.com/trends/explore?q=bmw+x6+2022&date=today+12-m"
},
... and other queries
]
},
{
"searchQuery":"Mercedes",
"top":[
{
"query":"mercedes benz",
"value":"100",
"extractedValue":100,
"link":"https://trends.google.com/trends/explore?q=mercedes+benz&date=today+12-m"
},
... and other queries
],
"rising":[
{
"query":"mercedes f1 2022",
"value":"+2,250%",
"extractedValue":2250,
"link":"https://trends.google.com/trends/explore?q=mercedes+f1+2022&date=today+12-m"
},
... and other queries
]
}
],
"relatedTopics":{
"top":[
{
"topic":{
"title":"Mercedes-Benz",
"type":"Luxury vehicles company"
},
"value":"100",
"extractedValue":100,
"link":"https://trends.google.com/trends/explore?q=/m/052mx&date=today+12-m"
},
...and other topics
],
"rising":[
{
"topic":{
"title":"Mercedes-Benz EQB",
"type":"SUV"
},
"value":"+700%",
"extractedValue":700,
"link":"https://trends.google.com/trends/explore?q=/g/11h__y1vw4&date=today+12-m"
},
...and other topics
]
}
}
Using Google Trends API
The difference is that you don’t need to use browser automation to scrape results, and write the parser from scratch and maintain it, which saves a lot of time.
There’s also a chance that the request might be blocked at some point from Google. Instead, you just need to iterate the structured JSON and get the data you want.
First, we need to install google-search-results-nodejs
. To do this you need to enter in your console: npm i google-search-results-nodejs
Here’s the full code example, if you don’t need an explanation:
const SerpApi = require("google-search-results-nodejs");
const search = new SerpApi.GoogleSearch(process.env.API_KEY); //your API key from serpapi.comconst searchQueries = "Mercedes"; // what we want to search (for interestOverTime, interestByRegion, relatedQueries, relatedTopics)
// const searchQueries = "Mercedes, BMW, Audi"; // what we want to search (for interestOverTime, comparedByRegion)const dataTypesMultiple = ["TIMESERIES", "GEO_MAP"];
const dataTypesSingle = ["TIMESERIES", "GEO_MAP_0", "RELATED_TOPICS", "RELATED_QUERIES"];const params = {
engine: "google_trends", // search engine
q: searchQueries, // search query
};const getJson = () => {
return new Promise((resolve) => {
search.json(params, resolve);
});
};const getResults = async () => {
const trendsResults = {};
if (searchQueries.split(",").length > 1) {
for (type of dataTypesMultiple) {
params.data_type = type;
const searchResult = await getJson();
if (type === "TIMESERIES") trendsResults.interestOverTime = searchResult.interest_over_time;
else if (type === "GEO_MAP") trendsResults.comparedByRegion = searchResult.compared_breakdown_by_region;
}
} else {
for (type of dataTypesSingle) {
params.data_type = type;
const searchResult = await getJson();
if (type === "TIMESERIES") trendsResults.interestOverTime = searchResult.interest_over_time;
else if (type === "GEO_MAP_0") trendsResults.interestByRegion = searchResult.interest_by_region;
else if (type === "RELATED_TOPICS") trendsResults.relatedTopics = searchResult.related_topics;
else if (type === "RELATED_QUERIES") trendsResults.relatedQueries = searchResult.related_queries;
}
}
return trendsResults;
};getResults().then((result) => console.dir(result, { depth: null }));
Code explanation
First, we need to declare SerpApi
from google-search-results-nodejs
library and define new search
instance with your API key from SerpApi:
const SerpApi = require("google-search-results-nodejs");
const search = new SerpApi.GoogleSearch(API_KEY);
Next, we write two different search queries to show you all available results (some of them are available only for a single search query, but some are available only for multiple search queries) and the necessary parameters for making a request :
const searchQueries = "Mercedes";
// const searchQueries = "Mercedes, BMW, Audi";const dataTypesMultiple = ["TIMESERIES", "GEO_MAP"];
const dataTypesSingle = ["TIMESERIES", "GEO_MAP_0", "RELATED_TOPICS", "RELATED_QUERIES"];const params = {
engine: "google_trends",
q: searchQueries,
};
Next, we wrap the search method from the SerpApi library in a promise to further work with the search results:
const getJson = () => {
return new Promise((resolve) => {
search.json(params, resolve);
});
};
And finally, we declare the function getResult
that gets data from the page and return it:
const getResults = async () => {
...
};
In this function first, we declare an object trendsResults
with results data:
const trendsResults = {};
Next, we need to define if searchQueries
has one or more values. If it has more than one, we can get interestOverTime
and comparedByRegion
data. To do this, we need to iterate over dataTypesMultiple
array (with the for...of
loop) and set data_type
value in the params object. Then we just receive searchResult
and select the necessary data:
if (searchQueries.split(",").length > 1) {
for (type of dataTypesMultiple) {
params.data_type = type;
const searchResult = await getJson();
if (type === "TIMESERIES") trendsResults.interestOverTime = searchResult.interest_over_time;
else if (type === "GEO_MAP") trendsResults.comparedByRegion = searchResult.compared_breakdown_by_region;
}
}
Otherwise, we iterate over dataTypesSingle
array and get all data from the single query search request:
else {
for (type of dataTypesSingle) {
params.data_type = type;
const searchResult = await getJson();
if (type === "TIMESERIES") trendsResults.interestOverTime = searchResult.interest_over_time;
else if (type === "GEO_MAP_0") trendsResults.interestByRegion = searchResult.interest_by_region;
else if (type === "RELATED_TOPICS") trendsResults.relatedTopics = searchResult.related_topics;
else if (type === "RELATED_QUERIES") trendsResults.relatedQueries = searchResult.related_queries;
}
}
After, we run the getResults
function and print all the received information in the console with the console.dir
method, which allows you to use an object with the necessary parameters to change default output options. Watch Node.js documentation for more info:
getResults().then((result) => console.dir(result, { depth: null }));
Output
📌Note: I’ve combined the results of different runs of our parser into one for convenience.
{
"interestOverTime":{
"timeline_data":[
{
"date":"Aug 29 – Sep 4, 2021",
"values":[
{
"query":"Mercedes",
"value":"74",
"extracted_value":74
},
{
"query":"BMW",
"value":"100",
"extracted_value":100
},
{
"query":"Audi",
"value":"60",
"extracted_value":60
}
]
},
... and other dates
],
"averages":[
{
"query":"Mercedes",
"value":66
},
{
"query":"BMW",
"value":90
},
{
"query":"Audi",
"value":55
}
]
},
"comparedByRegion":[
{
"geo":"LT",
"location":"Lithuania",
"max_value_index":1,
"values":[
{
"query":"Mercedes",
"value":"13%",
"extracted_value":13
},
{
"query":"BMW",
"value":"52%",
"extracted_value":52
},
{
"query":"Audi",
"value":"35%",
"extracted_value":35
}
]
},
...and other regions
],
"interestByRegion":[
{
"geo":"AL",
"location":"Albania",
"max_value_index":0,
"value":"100",
"extracted_value":100
},
...and other regions
],
"relatedTopics":{
"rising":[
{
"topic":{
"value":"/g/11h__y1vw4",
"title":"Mercedes-Benz EQB",
"type":"SUV"
},
"value":"+700%",
"extracted_value":700,
"link":"https://trends.google.com/trends/explore?q=/g/11h__y1vw4&date=today+12-m",
"serpapi_link":"https://serpapi.com/search.json?data_type=RELATED_TOPICS&date=today+12-m&engine=google_trends&q=%2Fg%2F11h__y1vw4&tz=420"
},
... and other topics
],
"top":[
{
"topic":{
"value":"/m/052mx",
"title":"Mercedes-Benz",
"type":"Luxury vehicles company"
},
"value":"100",
"extracted_value":100,
"link":"https://trends.google.com/trends/explore?q=/m/052mx&date=today+12-m",
"serpapi_link":"https://serpapi.com/search.json?data_type=RELATED_TOPICS&date=today+12-m&engine=google_trends&q=%2Fm%2F052mx&tz=420"
},
... and other topics
]
},
"relatedQueries":{
"rising":[
{
"query":"mercedes eqxx",
"value":"+2,450%",
"extracted_value":2450,
"link":"https://trends.google.com/trends/explore?q=mercedes+eqxx&date=today+12-m",
"serpapi_link":"https://serpapi.com/search.json?data_type=RELATED_QUERIES&date=today+12-m&engine=google_trends&q=mercedes+eqxx&tz=420"
},
... and other queries
],
"top":[
{
"query":"mercedes benz",
"value":"100",
"extracted_value":100,
"link":"https://trends.google.com/trends/explore?q=mercedes+benz&date=today+12-m",
"serpapi_link":"https://serpapi.com/search.json?data_type=RELATED_QUERIES&date=today+12-m&engine=google_trends&q=mercedes+benz&tz=420"
},
... and other queries
]
}
}
Links
If you want to see some projects made with SerpApi, please write me a message.
Add a Feature Request💫 or a Bug🐞