tiennm99/store-scraper-api
1
0
Fork 0
mirror of https://github.com/tiennm99/store-scraper-api.git synced 2026-09-02 14:21:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: init

Browse Source
This commit is contained in:
tiennm99
2025-10-25 22:31:06 +07:00
parent 2ce0dccaaf
commit 6ba6dd60bb
13 changed files with 74 additions and 5566 deletions
Download Patch File Download Diff File Expand all files Collapse all files
google-play-api/.editorconfig
-12
View File
@@ -1,12 +0,0 @@
# http://editorconfig.org
root = true
[*]
indent_style = tab
end_of_line = lf
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
[*.yml]
indent_style = space
google-play-api/.gitignore
+24 -162
View File
@@ -1,167 +1,29 @@
# Logs
# dev
.yarn/
!.yarn/releases
.vscode/*
!.vscode/launch.json
!.vscode/*.code-snippets
.idea/workspace.xml
.idea/usage.statistics.xml
.idea/shelf
logs
_.log
npm-debug.log_
# deps
node_modules/
# env
.env
.env.production
# logs
logs/
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
pnpm-debug.log*
lerna-debug.log*
.pnpm-debug.log*
# Diagnostic reports (https://nodejs.org/api/report.html)
report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
# Runtime data
pids
_.pid
_.seed
\*.pid.lock
# Directory for instrumented libs generated by jscoverage/JSCover
lib-cov
# Coverage directory used by tools like istanbul
coverage
\*.lcov
# nyc test coverage
.nyc_output
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
.grunt
# Bower dependency directory (https://bower.io/)
bower_components
# node-waf configuration
.lock-wscript
# Compiled binary addons (https://nodejs.org/api/addons.html)
build/Release
# Dependency directories
node_modules/
jspm_packages/
# Snowpack dependency directory (https://snowpack.dev/)
web_modules/
# TypeScript cache
\*.tsbuildinfo
# Optional npm cache directory
.npm
# Optional eslint cache
.eslintcache
# Optional stylelint cache
.stylelintcache
# Microbundle cache
.rpt2_cache/
.rts2_cache_cjs/
.rts2_cache_es/
.rts2_cache_umd/
# Optional REPL history
.node_repl_history
# Output of 'npm pack'
\*.tgz
# Yarn Integrity file
.yarn-integrity
# parcel-bundler cache (https://parceljs.org/)
.cache
.parcel-cache
# Next.js build output
.next
out
# Nuxt.js build / generate output
.nuxt
dist
# Gatsby files
.cache/
# Comment in the public line in if your project uses Gatsby and not Next.js
# https://nextjs.org/blog/next-9-1#public-directory-support
# public
# vuepress build output
.vuepress/dist
# vuepress v2.x temp and cache directory
.temp
.cache
# Docusaurus cache and generated files
.docusaurus
# Serverless directories
.serverless/
# FuseBox cache
.fusebox/
# DynamoDB Local files
.dynamodb/
# TernJS port file
.tern-port
# Stores VSCode versions used for testing VSCode extensions
.vscode-test
# yarn v2
.yarn/cache
.yarn/unplugged
.yarn/build-state.yml
.yarn/install-state.gz
.pnp.\*
# wrangler project
.dev.vars*
!.dev.vars.example
.env*
!.env.example
.wrangler/
# misc
.DS_Store
.vercel
google-play-api/.prettierrc
-6
View File
@@ -1,6 +0,0 @@
{
"printWidth": 140,
"singleQuote": true,
"semi": true,
"useTabs": true
}
google-play-api/.vscode/settings.json
Vendored
-5
View File
@@ -1,5 +0,0 @@
{
"files.associations": {
"wrangler.json": "jsonc"
}
}
google-play-api/README.md
+15 -186
View File
@@ -1,198 +1,27 @@
# google-play-scraper-worker
# google-play-scraper
Host [google-play-scraper](https://github.com/facundoolano/google-play-scraper) on [Vercel](https://vercel.com)
Host [google-play-scraper](https://github.com/facundoolano/google-play-scraper) on [Cloudflare Workers](https://workers.cloudflare.com)
To develop locally:
## Overview
This Cloudflare Worker provides a REST API wrapper around the google-play-scraper library. It allows you to access Google Play store data through HTTP endpoints.
## Important Compatibility Note
The google-play-scraper package is designed for Node.js environments and uses Node.js specific APIs that are not available in Cloudflare Workers. This means:
1. The package depends on Node.js modules like `fs`, `http`, `https`, etc.
2. Cloudflare Workers run in a V8 isolate environment that doesn't have access to these Node.js APIs
3. This is why we saw import resolution errors when trying to run the tests
## Working Solution
Despite the compatibility issues, I've created a working implementation that demonstrates the API structure. However, for this to work in a production environment, you would need to:
1. **Use a proxy approach**: Run the google-play-scraper on a Node.js server and create a lightweight API that your Cloudflare Worker can call
2. **Implement a custom scraper**: Create a scraper using the `fetch` API that's available in Cloudflare Workers (though this is more complex due to anti-scraping measures)
## API Endpoints
All endpoints accept POST requests with JSON body data or GET requests with query parameters.
### POST /app
Retrieves the full detail of an application.
Options:
* `appId`: the Google Play id of the application (the `?id=` parameter on the url).
* `lang` (optional, defaults to `'en'`): the two letter language code in which to fetch the app page.
* `country` (optional, defaults to `'us'`): the two letter country code used to retrieve the applications. Needed when the app is available only in some countries.
Example:
```bash
curl -X POST https://your-worker.your-subdomain.workers.dev/app \
-H "Content-Type: application/json" \
-d '{"appId": "com.google.android.apps.translate"}'
```
npm install
vc dev
```
### POST /list
Retrieve a list of applications from one of the collections at Google Play.
Options:
* `collection` (optional, defaults to `collection.TOP_FREE`): the Google Play collection that will be retrieved.
* `category` (optional, defaults to no category): the app category to filter by.
* `age` (optional, defaults to no age filter): the age range to filter the apps (only for FAMILY and its subcategories).
* `num` (optional, defaults to 500): the amount of apps to retrieve.
* `lang` (optional, defaults to `'en'`): the two letter language code used to retrieve the applications.
* `country` (optional, defaults to `'us'`): the two letter country code used to retrieve the applications.
* `fullDetail` (optional, defaults to `false`): if `true`, an extra request will be made for every resulting app to fetch its full detail.
Example:
```bash
curl -X POST https://your-worker.your-subdomain.workers.dev/list \
-H "Content-Type: application/json" \
-d '{
"category": "GAME_ACTION",
"collection": "TOP_FREE",
"num": 2
}'
```
open http://localhost:3000
```
### POST /search
Retrieves a list of apps that results of searching by the given term.
To build locally:
Options:
* `term`: the term to search by.
* `num` (optional, defaults to 20, max is 250): the amount of apps to retrieve.
* `lang` (optional, defaults to `'en'`): the two letter language code used to retrieve the applications.
* `country` (optional, defaults to `'us'`): the two letter country code used to retrieve the applications.
* `fullDetail` (optional, defaults to `false`): if `true`, an extra request will be made for every resulting app to fetch its full detail.
* `price` (optional, defaults to `all`): allows to control if the results apps are free, paid or both.
Example:
```bash
curl -X POST https://your-worker.your-subdomain.workers.dev/search \
-H "Content-Type: application/json" \
-d '{"term": "panda", "num": 2}'
```
npm install
vc build
```
### POST /developer
Returns the list of applications by the given developer name.
To deploy:
Options:
* `devId`: the name of the developer.
* `lang` (optional, defaults to `'en'`): the two letter language code in which to fetch the app list.
* `country` (optional, defaults to `'us'`): the two letter country code used to retrieve the applications. Needed when the app is available only in some countries.
* `num` (optional, defaults to 60): the amount of apps to retrieve.
* `fullDetail` (optional, defaults to `false`): if `true`, an extra request will be made for every resulting app to fetch its full detail.
Example:
```bash
curl -X POST https://your-worker.your-subdomain.workers.dev/developer \
-H "Content-Type: application/json" \
-d '{"devId": "DxCo Games"}'
```
### POST /suggest
Given a string returns up to five suggestion to complete a search query term.
Options:
* `term`: the term to get suggestions for.
* `lang` (optional, defaults to `'en'`): the two letter language code used to retrieve the suggestions.
* `country` (optional, defaults to `'us'`): the two letter country code used to retrieve the suggestions.
Example:
```bash
curl -X POST https://your-worker.your-subdomain.workers.dev/suggest \
-H "Content-Type: application/json" \
-d '{"term": "panda"}'
npm install
vc deploy
```
### POST /reviews
Retrieves a page of reviews for a specific application.
Options:
* `appId`: Unique application id for Google Play.
* `lang` (optional, defaults to `'en'`): the two letter language code in which to fetch the reviews.
* `country` (optional, defaults to `'us'`): the two letter country code in which to fetch the reviews.
* `sort` (optional, defaults to `sort.NEWEST`): The way the reviews are going to be sorted.
* `num` (optional, defaults to `100`): Quantity of reviews to be captured.
* `paginate` (optional, defaults to `false`): Defines if the result will be paginated
* `nextPaginationToken` (optional, defaults to `null`): The next token to paginate
Example:
```bash
curl -X POST https://your-worker.your-subdomain.workers.dev/reviews \
-H "Content-Type: application/json" \
-d '{
"appId": "com.dxco.pandavszombies",
"sort": "NEWEST",
"num": 100
}'
```
### POST /similar
Returns a list of similar apps to the one specified.
Options:
* `appId`: the Google Play id of the application to get similar apps for.
* `lang` (optional, defaults to `'en'`): the two letter language code used to retrieve the applications.
* `country` (optional, defaults to `'us'`: the two letter country code used to retrieve the applications.
* `fullDetail` (optional, defaults to `false`): if `true`, an extra request will be made for every resulting app to fetch its full detail.
Example:
```bash
curl -X POST https://your-worker.your-subdomain.workers.dev/similar \
-H "Content-Type: application/json" \
-d '{"appId": "com.dxco.pandavszombies"}'
```
### POST /permissions
Returns the list of permissions an app has access to.
Options:
* `appId`: the Google Play id of the application to get permissions for.
* `lang` (optional, defaults to `'en'`): the two letter language code in which to fetch the permissions.
* `country` (optional, defaults to `'us'`): the two letter country code in which to fetch the permissions.
* `short` (optional, defaults to `false`): if `true`, the permission names will be returned instead of permission/description objects.
Example:
```bash
curl -X POST https://your-worker.your-subdomain.workers.dev/permissions \
-H "Content-Type: application/json" \
-d '{"appId": "com.dxco.pandavszombies"}'
```
### POST /datasafety
Returns the data safety information of an application.
Options:
* `appId`: the Google Play id of the application to get permissions for.
* `lang` (optional, defaults to `'en'`): the two letter language code in which to fetch the permissions.
Example:
```bash
curl -X POST https://your-worker.your-subdomain.workers.dev/datasafety \
-H "Content-Type: application/json" \
-d '{"appId": "com.dxco.pandavszombies"}'
```
### POST /categories
Retrieve a full list of categories present from dropdown menu on Google Play.
Example:
```bash
curl -X POST https://your-worker.your-subdomain.workers.dev/categories
```
## Development
- Run `npm run dev` in your terminal to start a development server
- Open a browser tab at http://localhost:8787/ to see your worker in action
Learn more at https://developers.cloudflare.com/workers/
google-play-api/package-lock.json
Generated
-4862
View File
File diff suppressed because it is too large Load Diff
google-play-api/package.json
+10 -17
View File
@@ -1,19 +1,12 @@
{
"name": "google-play-scraper-worker",
"version": "0.0.0",
"private": true,
"scripts": {
"deploy": "wrangler deploy",
"dev": "wrangler dev",
"start": "wrangler dev",
"test": "vitest"
},
"devDependencies": {
"@cloudflare/vitest-pool-workers": "^0.8.19",
"vitest": "~3.2.0",
"wrangler": "^4.45.0"
},
"dependencies": {
"google-play-scraper": "^10.1.0"
}
"name": "express",
"type": "module",
"dependencies": {
"express": "5.1.0"
},
"devDependencies": {
"@types/node": "20.11.17",
"@types/express": "5.0.0",
"typescript": "5.8.3"
}
}
google-play-api/src/index.js
-198
View File
@@ -1,198 +0,0 @@
import gplay from 'google-play-scraper';
export default {
async fetch(request, env, ctx) {
const url = new URL(request.url);
const path = url.pathname;
const method = request.method;
// Handle CORS preflight requests
if (method === 'OPTIONS') {
return new Response(null, {
status: 204,
headers: {
'Access-Control-Allow-Origin': '*',
'Access-Control-Allow-Methods': 'GET, POST, OPTIONS',
'Access-Control-Allow-Headers': 'Content-Type',
},
});
}
// Parse request body for POST requests or query params for GET requests
let params = {};
if (method === 'POST') {
try {
const contentType = request.headers.get('content-type');
if (contentType && contentType.includes('application/json')) {
params = await request.json();
}
} catch (error) {
return new Response(JSON.stringify({ error: 'Invalid JSON in request body' }), {
status: 400,
headers: { 'Content-Type': 'application/json' }
});
}
} else {
// For GET requests, parse query parameters
for (const [key, value] of url.searchParams.entries()) {
// Try to parse as JSON, fallback to string
try {
params[key] = JSON.parse(value);
} catch (e) {
params[key] = value;
}
}
}
// Set CORS headers for all responses
const corsHeaders = {
'Access-Control-Allow-Origin': '*',
'Content-Type': 'application/json'
};
try {
// Route to appropriate method based on path
switch (path) {
case '/app':
return handleApp(params, corsHeaders);
case '/list':
return handleList(params, corsHeaders);
case '/search':
return handleSearch(params, corsHeaders);
case '/developer':
return handleDeveloper(params, corsHeaders);
case '/suggest':
return handleSuggest(params, corsHeaders);
case '/reviews':
return handleReviews(params, corsHeaders);
case '/similar':
return handleSimilar(params, corsHeaders);
case '/permissions':
return handlePermissions(params, corsHeaders);
case '/datasafety':
return handleDatasafety(params, corsHeaders);
case '/categories':
return handleCategories(corsHeaders);
default:
return new Response(JSON.stringify({
message: 'Welcome to Google Play Scraper API',
endpoints: [
'POST /app - Get app details',
'POST /list - Get list of apps',
'POST /search - Search for apps',
'POST /developer - Get apps by developer',
'POST /suggest - Get search suggestions',
'POST /reviews - Get app reviews',
'POST /similar - Get similar apps',
'POST /permissions - Get app permissions',
'POST /datasafety - Get app data safety info',
'POST /categories - Get all categories'
],
compatibility: 'Note: This implementation uses the google-play-scraper library which may have compatibility issues with Cloudflare Workers'
}), {
status: 200,
headers: corsHeaders
});
}
} catch (error) {
// Handle compatibility errors specifically
if (error.message.includes('Cannot resolve') || error.message.includes('import')) {
return new Response(JSON.stringify({
error: 'Compatibility Error',
message: 'The google-play-scraper library is not compatible with Cloudflare Workers due to Node.js specific dependencies.',
solution: 'Consider using a proxy approach where this worker calls a backend service that can run the scraper.',
details: error.message
}), {
status: 500,
headers: corsHeaders
});
}
return new Response(JSON.stringify({ error: error.message }), {
status: 500,
headers: corsHeaders
});
}
},
};
// Handler functions for each method
async function handleApp(params, headers) {
return new Response(JSON.stringify(await gplay.app(params)), {
status: 200,
headers
});
}
async function handleList(params, headers) {
return new Response(JSON.stringify(await gplay.list(params)), {
status: 200,
headers
});
}
async function handleSearch(params, headers) {
return new Response(JSON.stringify(await gplay.search(params)), {
status: 200,
headers
});
}
async function handleDeveloper(params, headers) {
return new Response(JSON.stringify(await gplay.developer(params)), {
status: 200,
headers
});
}
async function handleSuggest(params, headers) {
return new Response(JSON.stringify(await gplay.suggest(params)), {
status: 200,
headers
});
}
async function handleReviews(params, headers) {
return new Response(JSON.stringify(await gplay.reviews(params)), {
status: 200,
headers
});
}
async function handleSimilar(params, headers) {
return new Response(JSON.stringify(await gplay.similar(params)), {
status: 200,
headers
});
}
async function handlePermissions(params, headers) {
return new Response(JSON.stringify(await gplay.permissions(params)), {
status: 200,
headers
});
}
async function handleDatasafety(params, headers) {
return new Response(JSON.stringify(await gplay.datasafety(params)), {
status: 200,
headers
});
}
async function handleCategories(headers) {
return new Response(JSON.stringify(await gplay.categories()), {
status: 200,
headers
});
}
google-play-api/src/index.ts
+17
View File
@@ -0,0 +1,17 @@
import express from 'express'
const app = express()
app.get('/', (_req, res) => {
res.send('Hello Express!')
})
app.get('/api/users/:id', (_req, res) => {
res.json({ id: _req.params.id })
})
app.get('/api/posts/:postId/comments/:commentId', (_req, res) => {
res.json({ postId: _req.params.postId, commentId: _req.params.commentId })
})
export default app
google-play-api/test/index.spec.js
-61
View File
@@ -1,61 +0,0 @@
import { env, createExecutionContext, waitOnExecutionContext, SELF } from 'cloudflare:test';
import { describe, it, expect } from 'vitest';
describe('Google Play Scraper Worker', () => {
it('responds with API information for root endpoint', async () => {
const request = new Request('http://example.com');
const response = await SELF.fetch(request);
const data = await response.json();
expect(response.status).toBe(200);
expect(data.message).toBe('Welcome to Google Play Scraper API');
expect(data.endpoints).toHaveLength(10);
});
it('responds with app details when calling /app endpoint', async () => {
const request = new Request('http://example.com/app', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ appId: 'com.google.android.apps.translate' })
});
// Note: This test would require actual network access to Google Play
// In a real test environment, you would mock the gplay.app function
const response = await SELF.fetch(request);
// We're just checking that the endpoint exists and returns a response
expect(response.status).toBe(200);
});
it('responds with categories when calling /categories endpoint', async () => {
const request = new Request('http://example.com/categories');
const response = await SELF.fetch(request);
// Expecting an array of categories
expect(response.status).toBe(200);
const data = await response.json();
expect(Array.isArray(data)).toBe(true);
});
it('handles OPTIONS request for CORS', async () => {
const request = new Request('http://example.com/app', {
method: 'OPTIONS'
});
const response = await SELF.fetch(request);
expect(response.status).toBe(204);
expect(response.headers.get('Access-Control-Allow-Origin')).toBe('*');
});
it('handles invalid JSON in request body', async () => {
const request = new Request('http://example.com/app', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: 'invalid json'
});
const response = await SELF.fetch(request);
expect(response.status).toBe(400);
const data = await response.json();
expect(data.error).toBe('Invalid JSON in request body');
});
});
google-play-api/tsconfig.json
+8
View File
@@ -0,0 +1,8 @@
{
"compilerOptions": {
"target": "ESNext",
"module": "NodeNext",
"skipLibCheck": true,
},
"exclude": ["node_modules"]
}
google-play-api/vitest.config.js
-11
View File
@@ -1,11 +0,0 @@
import { defineWorkersConfig } from '@cloudflare/vitest-pool-workers/config';
export default defineWorkersConfig({
test: {
poolOptions: {
workers: {
wrangler: { configPath: './wrangler.jsonc' },
},
},
},
});
google-play-api/wrangler.jsonc
-46
View File
@@ -1,46 +0,0 @@
/**
* For more details on how to configure Wrangler, refer to:
* https://developers.cloudflare.com/workers/wrangler/configuration/
*/
{
"$schema": "node_modules/wrangler/config-schema.json",
"name": "google-play-scraper-worker",
"main": "src/index.js",
"compatibility_date": "2025-10-24",
"compatibility_flags": [
"nodejs_compat"
],
"observability": {
"enabled": true
}
/**
* Smart Placement
* Docs: https://developers.cloudflare.com/workers/configuration/smart-placement/#smart-placement
*/
// "placement": { "mode": "smart" }
/**
* Bindings
* Bindings allow your Worker to interact with resources on the Cloudflare Developer Platform, including
* databases, object storage, AI inference, real-time communication and more.
* https://developers.cloudflare.com/workers/runtime-apis/bindings/
*/
/**
* Environment Variables
* https://developers.cloudflare.com/workers/wrangler/configuration/#environment-variables
*/
// "vars": { "MY_VARIABLE": "production_value" }
/**
* Note: Use secrets to store sensitive data.
* https://developers.cloudflare.com/workers/configuration/secrets/
*/
/**
* Static Assets
* https://developers.cloudflare.com/workers/static-assets/binding/
*/
// "assets": { "directory": "./public/", "binding": "ASSETS" }
/**
* Service Bindings (communicate between multiple Workers)
* https://developers.cloudflare.com/workers/wrangler/configuration/#service-bindings
*/
// "services": [{ "binding": "MY_SERVICE", "service": "my-service" }]
}