From 7c162d40296bc00e41dd071d64d6407c91ea7b69 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 1 Aug 2026 02:06:43 +0000 Subject: [PATCH] mongodb: fix the queries and the file the driver reads `benchmark.sh` pointed `BENCH_QUERIES_FILE` at `queries.txt`, which does not exist - the file is `queries.sql` - so the driver read no queries at all. `queries.sql` was derived from `queries.js` with `EJSON.stringify`, which turns a JavaScript regular expression literal into an empty document. Q20 to Q23 therefore lost their predicate: `{"$match": {"URL": /google/}}` was committed as `{"$match": {"URL": {}}}`, which matches nothing. They are spelled as the Extended JSON `$regularExpression` now. Two pipelines did not implement their SQL. Q37 filtered `URL <> ''` where the SQL filters `Title <> ''` - a copy of the `$match` of Q36 - and Q28 left out the `MIN(Referer)` the SQL selects and matched `(?:www.)?` where the SQL has `(?:www\.)?`. A date was written as a bare calendar day, `{"$date": "2013-07-01"}`, which is not valid Extended JSON; `mongosh` accepts it but other drivers refuse it. It is the full instant now. `queries.js` is removed: `queries.sql` is the one file the benchmark and the playground read, and keeping a second copy of the same pipelines by hand is what broke them. --- mongodb/README.md | 6 +- mongodb/benchmark.sh | 4 +- mongodb/queries.js | 706 ------------------------------------------- mongodb/queries.sql | 24 +- 4 files changed, 18 insertions(+), 722 deletions(-) delete mode 100644 mongodb/queries.js diff --git a/mongodb/README.md b/mongodb/README.md index d55c15baa8..88e87cab3f 100644 --- a/mongodb/README.md +++ b/mongodb/README.md @@ -8,8 +8,10 @@ It may seem unfair to other benches that don't have same preparation but working require that all because you can't just make arbitrary queries on big collections in mongo. If you search mongo analogues for your SQL queries then you can inspire by -`queries.js` but remember that these queries specially adapted to current benchmark -and may be work slower in your cases/environment. Especially if you have less than 100k docs. +`queries.sql`, which holds one aggregation pipeline per line in Extended JSON, in the same +order as the SQL of `../clickhouse/queries.sql`. Remember that these queries are specially +adapted to the current benchmark and may work slower in your cases/environment, especially +if you have less than 100k docs. --- diff --git a/mongodb/benchmark.sh b/mongodb/benchmark.sh index fee6836c31..b3cba397a2 100755 --- a/mongodb/benchmark.sh +++ b/mongodb/benchmark.sh @@ -1,5 +1,5 @@ #!/bin/bash -# aggregation pipelines (queries.txt, EJSON one-per-line) rather than SQL. +# This system runs aggregation pipelines (queries.sql, Extended JSON one per line) rather than SQL. export BENCH_DOWNLOAD_SCRIPT="download-hits-tsv" -export BENCH_QUERIES_FILE="queries.txt" +export BENCH_QUERIES_FILE="queries.sql" exec ../lib/benchmark-common.sh diff --git a/mongodb/queries.js b/mongodb/queries.js deleted file mode 100644 index 6d55945af4..0000000000 --- a/mongodb/queries.js +++ /dev/null @@ -1,706 +0,0 @@ -// SQL comment above each query is from the 'clickhouse' queries.sql file as a reference -queries = []; - -// Q0 -// SELECT COUNT(*) FROM hits; -// NOTE: $project need to enable covered index usage with enabled DB options -queries.push([ - { $project: { _id: 1 } }, - { $count: "c" } -]); - -// Q1 -// SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0; -queries.push([ - { $match: { AdvEngineID: { $ne: 0 } } }, - { $count: "c" } -]); - -// Q2 -// SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits; -queries.push([ - { - $group: { - _id: null, - sum_AdvEngineID: { $sum: "$AdvEngineID" }, - c: { $sum: 1 }, - avg_ResolutionWidth: { $avg: "$ResolutionWidth" }, - }, - }, -]); - -// Q3 -// SELECT AVG(UserID) FROM hits; -// REMARKS: Precision is lost without the $toDecimal -queries.push([ - { - $group: { - _id: null, - a: { $avg: { $toDecimal: "$UserID" } } - } - }, -]); - -// Q4 -// SELECT COUNT(DISTINCT UserID) FROM hits; -queries.push([{ $group: { _id: "$UserID" } }, { $count: "c" }]); - -// Q5 -// SELECT COUNT(DISTINCT SearchPhrase) FROM hits; -queries.push([{ $group: { _id: "$SearchPhrase" } }, { $count: "c" }]); - -// Q6 -// SELECT MIN(EventDate), MAX(EventDate) FROM hits; -// NOTE: depends on collectionName var -queries.push([ - { $sort: { EventDate: 1 } }, - { $limit: 1 }, - { - $unionWith: { - coll: collectionName, - pipeline: [ - { $sort: { EventDate: -1 } }, - { $limit: 1 }, - ] - } - }, - { $group: { _id: null, tmpArray: { $push: "$EventDate" } } }, - { - $project: { - min: { $arrayElemAt: ["$tmpArray", 0] }, - max: { $arrayElemAt: ["$tmpArray", 1] } - } - } -]); - -// Q7 -// SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC; -queries.push([ - { $match: { AdvEngineID: { $ne: 0 } } }, - { $group: { _id: "$AdvEngineID", c: { $sum: 1 } } }, - { $sort: { c: -1 } }, -]); - -// Q8 -// SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10; -queries.push([ - { $group: { _id: { RegionID: "$RegionID", UserID: "$UserID" } } }, - { $group: { _id: "$_id.RegionID", u: { $sum: 1 } } }, - { $sort: { u: -1 } }, - { $limit: 10 }, -]); - -// Q9 -// SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10; -// NOTE: $addToSet is extremely expensive in this case so better use rewrite query with $lookup -// to collection itself. Not this query depends on collection name `collectionName` - the var from main -// run script in field `from` inside lookup. -queries.push([ - { - $group: { - _id: "$RegionID", - sum_AdvEngineID: { $sum: "$AdvEngineID" }, - avg_ResolutionWidth: { $avg: "$ResolutionWidth" }, - c: { $sum: 1 }, - }, - }, - { $sort: { c: -1 } }, - { $limit: 10 }, - { - $lookup: { - from: collectionName, - let: { regionIdVar: "$_id" }, - pipeline: [ - { $match: { $expr: { $eq: ["$RegionID", "$$regionIdVar"] } } }, - { $group: { _id: "$UserID" } }, - { $count: "c" } - ], - as: "count_distinct_UserID" - } - }, - { - $set: { count_distinct_UserID: { $arrayElemAt: ["$count_distinct_UserID.c", 0] } } - } -]); - -// Q10 -// SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10; -queries.push([ - { $match: { MobilePhoneModel: { $ne: "" } } }, - { - $group: { - _id: { MobilePhoneModel: "$MobilePhoneModel", UserID: "$UserID" }, - }, - }, - { $group: { _id: "$_id.MobilePhoneModel", u: { $sum: 1 } } }, - { $sort: { u: -1 } }, - { $limit: 10 }, -]); - -// Q11 -// SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10; -queries.push([ - { $match: { MobilePhoneModel: { $ne: "" } } }, - { - $group: { - _id: { - MobilePhone: "$MobilePhone", - MobilePhoneModel: "$MobilePhoneModel", - UserID: "$UserID", - }, - }, - }, - { - $group: { - _id: { - MobilePhone: "$_id.MobilePhone", - MobilePhoneModel: "$_id.MobilePhoneModel", - }, - u: { $sum: 1 }, - }, - }, - { $sort: { u: -1 } }, - { $limit: 10 }, -]); - -// Q12 -// SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; -queries.push([ - { $match: { SearchPhrase: { $ne: "" } } }, - { $group: { _id: "$SearchPhrase", c: { $sum: 1 } } }, - { $sort: { c: -1 } }, - { $limit: 10 }, -]); - -// Q13 -// SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10; -queries.push([ - { $match: { SearchPhrase: { $ne: "" } } }, - { $group: { _id: { SearchPhrase: "$SearchPhrase", UserID: "$UserID" } } }, - { $group: { _id: "$_id.SearchPhrase", u: { $sum: 1 } } }, - { $sort: { u: -1 } }, - { $limit: 10 }, -]); - -// Q14 -// SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10; -// NOTE: concat as _id take less cpu compare to object with two fields. Here we can use it because of int field SearchEngineID -queries.push([ - { $match: { SearchPhrase: { $ne: "" } } }, - { - $group: { - _id: { - $concat: [ - "$SearchPhrase", - "|", - { $toString: "$SearchEngineID" }, - ] - }, - SearchPhrase: { $first: "$SearchPhrase" }, - SearchEngineID: { $first: "$SearchEngineID" }, - c: { $sum: 1 }, - }, - }, - { $sort: { c: -1 } }, - { $limit: 10 }, -]); - -// Q15 -// SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10; -queries.push([ - { $group: { _id: "$UserID", c: { $sum: 1 } } }, - { $sort: { c: -1 } }, - { $limit: 10 }, -]); - -// Q16 -// SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10; -queries.push([ - { - $group: { - _id: { - $concat: [ - "$SearchPhrase", - "|", - { $toString: "$UserID" }, - ] - }, - SearchPhrase: { $first: "$SearchPhrase" }, - UserID: { $first: "$UserID" }, - c: { $sum: 1 }, - }, - }, - { $sort: { c: -1 } }, - { $limit: 10 }, -]); - -// Q17 -// SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase LIMIT 10; -queries.push([ - { - $group: { - _id: { - $concat: [ - "$SearchPhrase", - "|", - { $toString: "$UserID" }, - ] - }, - SearchPhrase: { $first: "$SearchPhrase" }, - UserID: { $first: "$UserID" }, - c: { $sum: 1 }, - }, - }, - { $limit: 10 }, -]); - -// Q18 -// SELECT UserID, extract(minute FROM EventTime) AS m, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, m, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10; -queries.push([ - { - $group: { - _id: { UserID: "$UserID", SearchPhrase: "$SearchPhrase", m: { $minute: "$EventTime" } }, - c: { $sum: 1 }, - }, - }, - { $sort: { c: -1 } }, - { $limit: 10 }, -]); - -// Q19 -// SELECT UserID FROM hits WHERE UserID = 435090932899640449; -queries.push([ - { $match: { UserID: NumberLong("435090932899640449") } }, - { $project: { UserID: 1 } }, -]); - -// Q20 -// SELECT COUNT(*) FROM hits WHERE URL LIKE '%google%'; -queries.push([{ $match: { URL: /google/ } }, { $count: "c" }]); - -// Q21 -// SELECT SearchPhrase, MIN(URL), COUNT(*) AS c FROM hits WHERE URL LIKE '%google%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; -queries.push([ - { $match: { URL: /google/, SearchPhrase: { $ne: "" } } }, - { - $group: { - _id: "$SearchPhrase", - min_URL: { $min: "$URL" }, - c: { $sum: 1 }, - }, - }, - { $sort: { c: -1 } }, - { $limit: 10 }, -]); - -// Q22 -// SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT(DISTINCT UserID) FROM hits WHERE Title LIKE '%Google%' AND URL NOT LIKE '%.google.%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; -queries.push([ - { - $match: { - Title: /Google/, - URL: { $not: /\.google\./ }, - SearchPhrase: { $ne: "" }, - }, - }, - { - $group: { - _id: "$SearchPhrase", - count_distinct_UserID: { $addToSet: "$UserID" }, - min_Title: { $min: "$Title" }, - min_URL: { $min: "$URL" }, - c: { $sum: 1 }, - }, - }, - { $set: { count_distinct_UserID: { $size: "$count_distinct_UserID" } } }, - { $sort: { c: -1 } }, - { $limit: 10 }, -]); - -// Q23 -// SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10; -queries.push([ - { $match: { URL: /google/ } }, - { $sort: { EventTime: 1 } }, - { $limit: 10 }, -]); - -// Q24 -// SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime LIMIT 10; -queries.push([ - { $match: { SearchPhrase: { $ne: "" } } }, - { $project: { _id: 0, SearchPhrase: 1 } }, - { $sort: { EventTime: 1 } }, - { $limit: 10 }, -]); - -// Q25 -// SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY SearchPhrase LIMIT 10; -queries.push([ - { $match: { SearchPhrase: { $ne: "" } } }, - { $sort: { SearchPhrase: 1 } }, - { $project: { SearchPhrase: 1 } }, - { $limit: 10 }, -]); - -// Q26 -// SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime, SearchPhrase LIMIT 10; -queries.push([ - { $match: { SearchPhrase: { $ne: "" } } }, - { $project: { _id: 0, EventTime: 1, SearchPhrase: 1 } }, - { $sort: { EventTime: 1, SearchPhrase: 1 } }, - { $limit: 10 }, -]); - -// Q27 -// SELECT CounterID, AVG(length(URL)) AS l, COUNT(*) AS c FROM hits WHERE URL <> '' GROUP BY CounterID HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; -// REMARK: 'LENGTH' seems to be specified as length in bytes, so translated to $strLenBytes, length in characters would be $strLenCP -queries.push([ - { $match: { URL: { $ne: "" } } }, - { - $group: { - _id: "$CounterID", - l: { $avg: { $strLenBytes: "$URL" } }, - c: { $sum: 1 }, - }, - }, - { $match: { c: { $gt: 100000 } } }, - { $sort: { l: -1 } }, - { $limit: 25 }, -]); - -// Q28 -// SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, AVG(length(Referer)) AS l, COUNT(*) AS c, MIN(Referer) FROM hits WHERE Referer <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; -// REMARK: This seems right but unsure of correct output for this one (mysql results seem strange) -queries.push([ - { $match: { Referer: { $ne: "" } } }, - { $project: { _id: 0, Referer: 1 } }, - { - $set: { - k: { - $regexFind: { - input: "$Referer", - regex: "^https?://(?:www.)?([^/]+)/.*$", - }, - }, - }, - }, - { - $group: { - _id: { $ifNull: [{ $first: "$k.captures" }, "$Referer"] }, - l: { $avg: { $strLenBytes: "$Referer" } }, - c: { $sum: 1 }, - }, - }, - { $match: { c: { $gt: 100000 } } }, - { $sort: { l: -1 } }, - { $limit: 25 }, -]); - -// Q29 -// SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), ..., SUM(ResolutionWidth + 89) FROM hits; -let sums = { _id: null }; -for (i = 0; i < 90; ++i) { - sums["srw_plus_" + i] = { - $sum: { $add: ["$ResolutionWidth", i] }, - }; -} -queries.push([ - { $project: { _id: 0, ResolutionWidth: { "$toLong": "$ResolutionWidth" } } }, - { $group: sums } -]); - -// Q30 -// SELECT SearchEngineID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10; -queries.push([ - { $match: { SearchPhrase: { $ne: "" } } }, - { - $group: { - _id: { - $concat: [ - { $toString: "$SearchEngineID" }, - "|", - { $toString: "$ClientIP" }, - ] - }, - SearchEngineID: { $first: "$SearchEngineID" }, - ClientIP: { $first: "$ClientIP" }, - avg_ResolutionWidth: { $avg: "$ResolutionWidth" }, - sum_IsRefresh: { $sum: "$IsRefresh" }, - c: { $sum: 1 }, - }, - }, - { $sort: { c: -1 } }, - { $limit: 10 }, -]); - -// Q31 -// SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10; -queries.push([ - { $match: { SearchPhrase: { $ne: "" } } }, - { - $group: { - _id: { - $concat: [ - { $toString: "$WatchID" }, - "|", - { $toString: "$ClientIP" }, - ] - }, - WatchID: { $first: "$WatchID" }, - ClientIP: { $first: "$ClientIP" }, - avg_ResolutionWidth: { $avg: "$ResolutionWidth" }, - sum_IsRefresh: { $sum: "$IsRefresh" }, - c: { $sum: 1 }, - }, - }, - { $sort: { c: -1 } }, - { $limit: 10 }, -]); - -// Q32 -// SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10; -queries.push([ - { - $group: { - _id: { - $concat: [ - { $toString: "$ClientIP" }, - "|", - { $toString: "$WatchID" }, - ] - }, - WatchID: { $first: "$WatchID" }, - ClientIP: { $first: "$ClientIP" }, - avg_ResolutionWidth: { $avg: "$ResolutionWidth" }, - sum_IsRefresh: { $sum: "$IsRefresh" }, - c: { $sum: 1 }, - }, - }, - { $sort: { c: -1 } }, - { $limit: 10 }, -]); - -// Q33 -// SELECT URL, COUNT(*) AS c FROM hits GROUP BY URL ORDER BY c DESC LIMIT 10; -queries.push([ - { $group: { _id: "$URL", c: { $sum: 1 } } }, - { $sort: { c: -1 } }, - { $limit: 10 }, -]); - -// Q34 -// SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC LIMIT 10; -queries.push([ - { $group: { _id: "$URL", c: { $sum: 1 } } }, - { $sort: { c: -1 } }, - { $limit: 10 }, - { $set: { one: 1 } } -]); - -// Q35 -// SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS c FROM hits GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY c DESC LIMIT 10; -queries.push([ - { - $group: { - _id: "$ClientIP", - c: { $sum: 1 }, - }, - }, - { $sort: { c: -1 } }, - { $limit: 10 }, - { - $set: { - ClientIP_0: "$_id", - ClientIP_1: { $add: ["$_id", -1] }, - ClientIP_2: { $add: ["$_id", -2] }, - ClientIP_3: { $add: ["$_id", -3] }, - } - } -]); - -// Q36 -// SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND URL <> '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10; -queries.push([ - { - $match: { - CounterID: 62, - EventDate: { $gte: ISODate("2013-07-01"), $lte: ISODate("2013-07-31") }, - DontCountHits: 0, - IsRefresh: 0, - URL: { $ne: "" }, - }, - }, - { - $group: { - _id: "$URL", - pageViews: { $sum: 1 }, - }, - }, - { $sort: { pageViews: -1 } }, - { $limit: 10 }, -]); - -// Q37 -// SELECT Title, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND Title <> '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10; -queries.push([ - { - $match: { - CounterID: 62, - EventDate: { $gte: ISODate("2013-07-01"), $lte: ISODate("2013-07-31") }, - DontCountHits: 0, - IsRefresh: 0, - URL: { $ne: "" }, - }, - }, - { - $group: { - _id: "$Title", - pageViews: { $sum: 1 }, - }, - }, - { $sort: { pageViews: -1 } }, - { $limit: 10 }, -]); - -// Q38 -// SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND IsLink <> 0 AND IsDownload = 0 GROUP BY URL ORDER BY PageViews DESC LIMIT 10 OFFSET 1000; -queries.push([ - { - $match: { - CounterID: 62, - EventDate: { $gte: ISODate("2013-07-01"), $lte: ISODate("2013-07-31") }, - IsRefresh: 0, - IsLink: { $ne: 0 }, - IsDownload: 0, - URL: { $ne: "" }, - }, - }, - { - $group: { - _id: "$Title", - pageViews: { $sum: 1 }, - }, - }, - { $sort: { pageViews: -1 } }, - { $skip: 1000 }, - { $limit: 10 }, -]); - -// Q39 -// SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEngineID = 0 AND AdvEngineID = 0) THEN Referer ELSE '' END AS Src, URL AS Dst, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000; -queries.push([ - { - $match: { - CounterID: 62, - EventDate: { $gte: ISODate("2013-07-01"), $lte: ISODate("2013-07-31") }, - IsRefresh: 0, - }, - }, - { - $set: { - Src: { - $cond: { - if: { - $and: [ - { $eq: ["$SearchEngineID", 0] }, - { $eq: ["$AdvEngineID", 0] }, - ], - }, - then: "$Referer", - else: "", - }, - }, - Dst: "$URL", - }, - }, - { - $group: { - _id: { - TraficSourceID: "$TraficSourceID", - SearchEngineID: "$SearchEngineID", - AdvEngineID: "$AdvEngineID", - Src: "$Src", - Dst: "$Dst", - }, - pageViews: { $sum: 1 }, - }, - }, - { $sort: { pageViews: -1 } }, - { $skip: 1000 }, - { $limit: 10 }, -]); - -// Q40 -// SELECT URLHash, EventDate, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND TraficSourceID IN (-1, 6) AND RefererHash = 3594120000172545465 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 10 OFFSET 100; -queries.push([ - { - $match: { - CounterID: 62, - EventDate: { $gte: ISODate("2013-07-01"), $lte: ISODate("2013-07-31") }, - IsRefresh: 0, - TraficSourceID: { $in: [-1, 6] }, - RefererHash: NumberLong("3594120000172545465"), - }, - }, - { - $group: { - _id: { - URLHash: "$URLHash", - EventDate: "$EventDate", - }, - pageViews: { $sum: 1 }, - }, - }, - { $sort: { pageViews: -1 } }, - { $skip: 100 }, - { $limit: 10 }, -]); - -// Q41 -// SELECT WindowClientWidth, WindowClientHeight, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND DontCountHits = 0 AND URLHash = 2868770270353813622 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10 OFFSET 10000; -queries.push([ - { - $match: { - CounterID: 62, - EventDate: { $gte: ISODate("2013-07-01"), $lte: ISODate("2013-07-31") }, - IsRefresh: 0, - DontCountHits: 0, - URLHash: NumberLong("2868770270353813622"), - }, - }, - { - $group: { - _id: { - WindowClientWidth: "$WindowClientWidth", - WindowClientHeight: "$WindowClientHeight", - }, - pageViews: { $sum: 1 }, - }, - }, - { $sort: { pageViews: -1 } }, - { $skip: 10000 }, - { $limit: 10 }, -]); - -// Q42 -// SELECT DATE_TRUNC('minute', EventTime) AS M, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-14' AND EventDate <= '2013-07-15' AND IsRefresh = 0 AND DontCountHits = 0 GROUP BY DATE_TRUNC('minute', EventTime) ORDER BY DATE_TRUNC('minute', EventTime) LIMIT 10 OFFSET 1000; -queries.push([ - { - $match: { - CounterID: 62, - EventDate: { $gte: ISODate("2013-07-14"), $lte: ISODate("2013-07-15") }, - IsRefresh: 0, - DontCountHits: 0, - }, - }, - { - $group: { - _id: { $dateTrunc: { date: "$EventTime", unit: "minute" } }, - pageViews: { $sum: 1 }, - }, - }, - { $sort: { _id: 1 } }, - { $skip: 1000 }, - { $limit: 10 }, -]); diff --git a/mongodb/queries.sql b/mongodb/queries.sql index 915f13421d..a384626ae5 100644 --- a/mongodb/queries.sql +++ b/mongodb/queries.sql @@ -18,15 +18,15 @@ [{"$group":{"_id":{"$concat":["$SearchPhrase","|",{"$toString":"$UserID"}]},"SearchPhrase":{"$first":"$SearchPhrase"},"UserID":{"$first":"$UserID"},"c":{"$sum":1}}},{"$limit":10}] [{"$group":{"_id":{"UserID":"$UserID","SearchPhrase":"$SearchPhrase","m":{"$minute":"$EventTime"}},"c":{"$sum":1}}},{"$sort":{"c":-1}},{"$limit":10}] [{"$match":{"UserID":{"$numberLong":"435090932899640449"}}},{"$project":{"UserID":1}}] -[{"$match":{"URL":{}}},{"$count":"c"}] -[{"$match":{"URL":{},"SearchPhrase":{"$ne":""}}},{"$group":{"_id":"$SearchPhrase","min_URL":{"$min":"$URL"},"c":{"$sum":1}}},{"$sort":{"c":-1}},{"$limit":10}] -[{"$match":{"Title":{},"URL":{"$not":{}},"SearchPhrase":{"$ne":""}}},{"$group":{"_id":"$SearchPhrase","count_distinct_UserID":{"$addToSet":"$UserID"},"min_Title":{"$min":"$Title"},"min_URL":{"$min":"$URL"},"c":{"$sum":1}}},{"$set":{"count_distinct_UserID":{"$size":"$count_distinct_UserID"}}},{"$sort":{"c":-1}},{"$limit":10}] -[{"$match":{"URL":{}}},{"$sort":{"EventTime":1}},{"$limit":10}] +[{"$match":{"URL":{"$regularExpression":{"pattern":"google","options":""}}}},{"$count":"c"}] +[{"$match":{"URL":{"$regularExpression":{"pattern":"google","options":""}},"SearchPhrase":{"$ne":""}}},{"$group":{"_id":"$SearchPhrase","min_URL":{"$min":"$URL"},"c":{"$sum":1}}},{"$sort":{"c":-1}},{"$limit":10}] +[{"$match":{"Title":{"$regularExpression":{"pattern":"Google","options":""}},"URL":{"$not":{"$regularExpression":{"pattern":"\\.google\\.","options":""}}},"SearchPhrase":{"$ne":""}}},{"$group":{"_id":"$SearchPhrase","count_distinct_UserID":{"$addToSet":"$UserID"},"min_Title":{"$min":"$Title"},"min_URL":{"$min":"$URL"},"c":{"$sum":1}}},{"$set":{"count_distinct_UserID":{"$size":"$count_distinct_UserID"}}},{"$sort":{"c":-1}},{"$limit":10}] +[{"$match":{"URL":{"$regularExpression":{"pattern":"google","options":""}}}},{"$sort":{"EventTime":1}},{"$limit":10}] [{"$match":{"SearchPhrase":{"$ne":""}}},{"$project":{"_id":0,"SearchPhrase":1}},{"$sort":{"EventTime":1}},{"$limit":10}] [{"$match":{"SearchPhrase":{"$ne":""}}},{"$sort":{"SearchPhrase":1}},{"$project":{"SearchPhrase":1}},{"$limit":10}] [{"$match":{"SearchPhrase":{"$ne":""}}},{"$project":{"_id":0,"EventTime":1,"SearchPhrase":1}},{"$sort":{"EventTime":1,"SearchPhrase":1}},{"$limit":10}] [{"$match":{"URL":{"$ne":""}}},{"$group":{"_id":"$CounterID","l":{"$avg":{"$strLenBytes":"$URL"}},"c":{"$sum":1}}},{"$match":{"c":{"$gt":100000}}},{"$sort":{"l":-1}},{"$limit":25}] -[{"$match":{"Referer":{"$ne":""}}},{"$project":{"_id":0,"Referer":1}},{"$set":{"k":{"$regexFind":{"input":"$Referer","regex":"^https?://(?:www.)?([^/]+)/.*$"}}}},{"$group":{"_id":{"$ifNull":[{"$first":"$k.captures"},"$Referer"]},"l":{"$avg":{"$strLenBytes":"$Referer"}},"c":{"$sum":1}}},{"$match":{"c":{"$gt":100000}}},{"$sort":{"l":-1}},{"$limit":25}] +[{"$match":{"Referer":{"$ne":""}}},{"$project":{"_id":0,"Referer":1}},{"$set":{"k":{"$regexFind":{"input":"$Referer","regex":"^https?://(?:www\\.)?([^/]+)/.*$"}}}},{"$group":{"_id":{"$ifNull":[{"$first":"$k.captures"},"$Referer"]},"l":{"$avg":{"$strLenBytes":"$Referer"}},"c":{"$sum":1},"min_Referer":{"$min":"$Referer"}}},{"$match":{"c":{"$gt":100000}}},{"$sort":{"l":-1}},{"$limit":25}] [{"$project":{"_id":0,"ResolutionWidth":{"$toLong":"$ResolutionWidth"}}},{"$group":{"_id":null,"srw_plus_0":{"$sum":{"$add":["$ResolutionWidth",0]}},"srw_plus_1":{"$sum":{"$add":["$ResolutionWidth",1]}},"srw_plus_2":{"$sum":{"$add":["$ResolutionWidth",2]}},"srw_plus_3":{"$sum":{"$add":["$ResolutionWidth",3]}},"srw_plus_4":{"$sum":{"$add":["$ResolutionWidth",4]}},"srw_plus_5":{"$sum":{"$add":["$ResolutionWidth",5]}},"srw_plus_6":{"$sum":{"$add":["$ResolutionWidth",6]}},"srw_plus_7":{"$sum":{"$add":["$ResolutionWidth",7]}},"srw_plus_8":{"$sum":{"$add":["$ResolutionWidth",8]}},"srw_plus_9":{"$sum":{"$add":["$ResolutionWidth",9]}},"srw_plus_10":{"$sum":{"$add":["$ResolutionWidth",10]}},"srw_plus_11":{"$sum":{"$add":["$ResolutionWidth",11]}},"srw_plus_12":{"$sum":{"$add":["$ResolutionWidth",12]}},"srw_plus_13":{"$sum":{"$add":["$ResolutionWidth",13]}},"srw_plus_14":{"$sum":{"$add":["$ResolutionWidth",14]}},"srw_plus_15":{"$sum":{"$add":["$ResolutionWidth",15]}},"srw_plus_16":{"$sum":{"$add":["$ResolutionWidth",16]}},"srw_plus_17":{"$sum":{"$add":["$ResolutionWidth",17]}},"srw_plus_18":{"$sum":{"$add":["$ResolutionWidth",18]}},"srw_plus_19":{"$sum":{"$add":["$ResolutionWidth",19]}},"srw_plus_20":{"$sum":{"$add":["$ResolutionWidth",20]}},"srw_plus_21":{"$sum":{"$add":["$ResolutionWidth",21]}},"srw_plus_22":{"$sum":{"$add":["$ResolutionWidth",22]}},"srw_plus_23":{"$sum":{"$add":["$ResolutionWidth",23]}},"srw_plus_24":{"$sum":{"$add":["$ResolutionWidth",24]}},"srw_plus_25":{"$sum":{"$add":["$ResolutionWidth",25]}},"srw_plus_26":{"$sum":{"$add":["$ResolutionWidth",26]}},"srw_plus_27":{"$sum":{"$add":["$ResolutionWidth",27]}},"srw_plus_28":{"$sum":{"$add":["$ResolutionWidth",28]}},"srw_plus_29":{"$sum":{"$add":["$ResolutionWidth",29]}},"srw_plus_30":{"$sum":{"$add":["$ResolutionWidth",30]}},"srw_plus_31":{"$sum":{"$add":["$ResolutionWidth",31]}},"srw_plus_32":{"$sum":{"$add":["$ResolutionWidth",32]}},"srw_plus_33":{"$sum":{"$add":["$ResolutionWidth",33]}},"srw_plus_34":{"$sum":{"$add":["$ResolutionWidth",34]}},"srw_plus_35":{"$sum":{"$add":["$ResolutionWidth",35]}},"srw_plus_36":{"$sum":{"$add":["$ResolutionWidth",36]}},"srw_plus_37":{"$sum":{"$add":["$ResolutionWidth",37]}},"srw_plus_38":{"$sum":{"$add":["$ResolutionWidth",38]}},"srw_plus_39":{"$sum":{"$add":["$ResolutionWidth",39]}},"srw_plus_40":{"$sum":{"$add":["$ResolutionWidth",40]}},"srw_plus_41":{"$sum":{"$add":["$ResolutionWidth",41]}},"srw_plus_42":{"$sum":{"$add":["$ResolutionWidth",42]}},"srw_plus_43":{"$sum":{"$add":["$ResolutionWidth",43]}},"srw_plus_44":{"$sum":{"$add":["$ResolutionWidth",44]}},"srw_plus_45":{"$sum":{"$add":["$ResolutionWidth",45]}},"srw_plus_46":{"$sum":{"$add":["$ResolutionWidth",46]}},"srw_plus_47":{"$sum":{"$add":["$ResolutionWidth",47]}},"srw_plus_48":{"$sum":{"$add":["$ResolutionWidth",48]}},"srw_plus_49":{"$sum":{"$add":["$ResolutionWidth",49]}},"srw_plus_50":{"$sum":{"$add":["$ResolutionWidth",50]}},"srw_plus_51":{"$sum":{"$add":["$ResolutionWidth",51]}},"srw_plus_52":{"$sum":{"$add":["$ResolutionWidth",52]}},"srw_plus_53":{"$sum":{"$add":["$ResolutionWidth",53]}},"srw_plus_54":{"$sum":{"$add":["$ResolutionWidth",54]}},"srw_plus_55":{"$sum":{"$add":["$ResolutionWidth",55]}},"srw_plus_56":{"$sum":{"$add":["$ResolutionWidth",56]}},"srw_plus_57":{"$sum":{"$add":["$ResolutionWidth",57]}},"srw_plus_58":{"$sum":{"$add":["$ResolutionWidth",58]}},"srw_plus_59":{"$sum":{"$add":["$ResolutionWidth",59]}},"srw_plus_60":{"$sum":{"$add":["$ResolutionWidth",60]}},"srw_plus_61":{"$sum":{"$add":["$ResolutionWidth",61]}},"srw_plus_62":{"$sum":{"$add":["$ResolutionWidth",62]}},"srw_plus_63":{"$sum":{"$add":["$ResolutionWidth",63]}},"srw_plus_64":{"$sum":{"$add":["$ResolutionWidth",64]}},"srw_plus_65":{"$sum":{"$add":["$ResolutionWidth",65]}},"srw_plus_66":{"$sum":{"$add":["$ResolutionWidth",66]}},"srw_plus_67":{"$sum":{"$add":["$ResolutionWidth",67]}},"srw_plus_68":{"$sum":{"$add":["$ResolutionWidth",68]}},"srw_plus_69":{"$sum":{"$add":["$ResolutionWidth",69]}},"srw_plus_70":{"$sum":{"$add":["$ResolutionWidth",70]}},"srw_plus_71":{"$sum":{"$add":["$ResolutionWidth",71]}},"srw_plus_72":{"$sum":{"$add":["$ResolutionWidth",72]}},"srw_plus_73":{"$sum":{"$add":["$ResolutionWidth",73]}},"srw_plus_74":{"$sum":{"$add":["$ResolutionWidth",74]}},"srw_plus_75":{"$sum":{"$add":["$ResolutionWidth",75]}},"srw_plus_76":{"$sum":{"$add":["$ResolutionWidth",76]}},"srw_plus_77":{"$sum":{"$add":["$ResolutionWidth",77]}},"srw_plus_78":{"$sum":{"$add":["$ResolutionWidth",78]}},"srw_plus_79":{"$sum":{"$add":["$ResolutionWidth",79]}},"srw_plus_80":{"$sum":{"$add":["$ResolutionWidth",80]}},"srw_plus_81":{"$sum":{"$add":["$ResolutionWidth",81]}},"srw_plus_82":{"$sum":{"$add":["$ResolutionWidth",82]}},"srw_plus_83":{"$sum":{"$add":["$ResolutionWidth",83]}},"srw_plus_84":{"$sum":{"$add":["$ResolutionWidth",84]}},"srw_plus_85":{"$sum":{"$add":["$ResolutionWidth",85]}},"srw_plus_86":{"$sum":{"$add":["$ResolutionWidth",86]}},"srw_plus_87":{"$sum":{"$add":["$ResolutionWidth",87]}},"srw_plus_88":{"$sum":{"$add":["$ResolutionWidth",88]}},"srw_plus_89":{"$sum":{"$add":["$ResolutionWidth",89]}}}}] [{"$match":{"SearchPhrase":{"$ne":""}}},{"$group":{"_id":{"$concat":[{"$toString":"$SearchEngineID"},"|",{"$toString":"$ClientIP"}]},"SearchEngineID":{"$first":"$SearchEngineID"},"ClientIP":{"$first":"$ClientIP"},"avg_ResolutionWidth":{"$avg":"$ResolutionWidth"},"sum_IsRefresh":{"$sum":"$IsRefresh"},"c":{"$sum":1}}},{"$sort":{"c":-1}},{"$limit":10}] [{"$match":{"SearchPhrase":{"$ne":""}}},{"$group":{"_id":{"$concat":[{"$toString":"$WatchID"},"|",{"$toString":"$ClientIP"}]},"WatchID":{"$first":"$WatchID"},"ClientIP":{"$first":"$ClientIP"},"avg_ResolutionWidth":{"$avg":"$ResolutionWidth"},"sum_IsRefresh":{"$sum":"$IsRefresh"},"c":{"$sum":1}}},{"$sort":{"c":-1}},{"$limit":10}] @@ -34,10 +34,10 @@ [{"$group":{"_id":"$URL","c":{"$sum":1}}},{"$sort":{"c":-1}},{"$limit":10}] [{"$group":{"_id":"$URL","c":{"$sum":1}}},{"$sort":{"c":-1}},{"$limit":10},{"$set":{"one":1}}] [{"$group":{"_id":"$ClientIP","c":{"$sum":1}}},{"$sort":{"c":-1}},{"$limit":10},{"$set":{"ClientIP_0":"$_id","ClientIP_1":{"$add":["$_id",-1]},"ClientIP_2":{"$add":["$_id",-2]},"ClientIP_3":{"$add":["$_id",-3]}}}] -[{"$match":{"CounterID":62,"EventDate":{"$gte":{"$date":"2013-07-01"},"$lte":{"$date":"2013-07-31"}},"DontCountHits":0,"IsRefresh":0,"URL":{"$ne":""}}},{"$group":{"_id":"$URL","pageViews":{"$sum":1}}},{"$sort":{"pageViews":-1}},{"$limit":10}] -[{"$match":{"CounterID":62,"EventDate":{"$gte":{"$date":"2013-07-01"},"$lte":{"$date":"2013-07-31"}},"DontCountHits":0,"IsRefresh":0,"URL":{"$ne":""}}},{"$group":{"_id":"$Title","pageViews":{"$sum":1}}},{"$sort":{"pageViews":-1}},{"$limit":10}] -[{"$match":{"CounterID":62,"EventDate":{"$gte":{"$date":"2013-07-01"},"$lte":{"$date":"2013-07-31"}},"IsRefresh":0,"IsLink":{"$ne":0},"IsDownload":0,"URL":{"$ne":""}}},{"$group":{"_id":"$Title","pageViews":{"$sum":1}}},{"$sort":{"pageViews":-1}},{"$skip":1000},{"$limit":10}] -[{"$match":{"CounterID":62,"EventDate":{"$gte":{"$date":"2013-07-01"},"$lte":{"$date":"2013-07-31"}},"IsRefresh":0}},{"$set":{"Src":{"$cond":{"if":{"$and":[{"$eq":["$SearchEngineID",0]},{"$eq":["$AdvEngineID",0]}]},"then":"$Referer","else":""}},"Dst":"$URL"}},{"$group":{"_id":{"TraficSourceID":"$TraficSourceID","SearchEngineID":"$SearchEngineID","AdvEngineID":"$AdvEngineID","Src":"$Src","Dst":"$Dst"},"pageViews":{"$sum":1}}},{"$sort":{"pageViews":-1}},{"$skip":1000},{"$limit":10}] -[{"$match":{"CounterID":62,"EventDate":{"$gte":{"$date":"2013-07-01"},"$lte":{"$date":"2013-07-31"}},"IsRefresh":0,"TraficSourceID":{"$in":[-1,6]},"RefererHash":{"$numberLong":"3594120000172545465"}}},{"$group":{"_id":{"URLHash":"$URLHash","EventDate":"$EventDate"},"pageViews":{"$sum":1}}},{"$sort":{"pageViews":-1}},{"$skip":100},{"$limit":10}] -[{"$match":{"CounterID":62,"EventDate":{"$gte":{"$date":"2013-07-01"},"$lte":{"$date":"2013-07-31"}},"IsRefresh":0,"DontCountHits":0,"URLHash":{"$numberLong":"2868770270353813622"}}},{"$group":{"_id":{"WindowClientWidth":"$WindowClientWidth","WindowClientHeight":"$WindowClientHeight"},"pageViews":{"$sum":1}}},{"$sort":{"pageViews":-1}},{"$skip":10000},{"$limit":10}] -[{"$match":{"CounterID":62,"EventDate":{"$gte":{"$date":"2013-07-14"},"$lte":{"$date":"2013-07-15"}},"IsRefresh":0,"DontCountHits":0}},{"$group":{"_id":{"$dateTrunc":{"date":"$EventTime","unit":"minute"}},"pageViews":{"$sum":1}}},{"$sort":{"_id":1}},{"$skip":1000},{"$limit":10}] +[{"$match":{"CounterID":62,"EventDate":{"$gte":{"$date":"2013-07-01T00:00:00Z"},"$lte":{"$date":"2013-07-31T00:00:00Z"}},"DontCountHits":0,"IsRefresh":0,"URL":{"$ne":""}}},{"$group":{"_id":"$URL","pageViews":{"$sum":1}}},{"$sort":{"pageViews":-1}},{"$limit":10}] +[{"$match":{"CounterID":62,"EventDate":{"$gte":{"$date":"2013-07-01T00:00:00Z"},"$lte":{"$date":"2013-07-31T00:00:00Z"}},"DontCountHits":0,"IsRefresh":0,"Title":{"$ne":""}}},{"$group":{"_id":"$Title","pageViews":{"$sum":1}}},{"$sort":{"pageViews":-1}},{"$limit":10}] +[{"$match":{"CounterID":62,"EventDate":{"$gte":{"$date":"2013-07-01T00:00:00Z"},"$lte":{"$date":"2013-07-31T00:00:00Z"}},"IsRefresh":0,"IsLink":{"$ne":0},"IsDownload":0,"URL":{"$ne":""}}},{"$group":{"_id":"$Title","pageViews":{"$sum":1}}},{"$sort":{"pageViews":-1}},{"$skip":1000},{"$limit":10}] +[{"$match":{"CounterID":62,"EventDate":{"$gte":{"$date":"2013-07-01T00:00:00Z"},"$lte":{"$date":"2013-07-31T00:00:00Z"}},"IsRefresh":0}},{"$set":{"Src":{"$cond":{"if":{"$and":[{"$eq":["$SearchEngineID",0]},{"$eq":["$AdvEngineID",0]}]},"then":"$Referer","else":""}},"Dst":"$URL"}},{"$group":{"_id":{"TraficSourceID":"$TraficSourceID","SearchEngineID":"$SearchEngineID","AdvEngineID":"$AdvEngineID","Src":"$Src","Dst":"$Dst"},"pageViews":{"$sum":1}}},{"$sort":{"pageViews":-1}},{"$skip":1000},{"$limit":10}] +[{"$match":{"CounterID":62,"EventDate":{"$gte":{"$date":"2013-07-01T00:00:00Z"},"$lte":{"$date":"2013-07-31T00:00:00Z"}},"IsRefresh":0,"TraficSourceID":{"$in":[-1,6]},"RefererHash":{"$numberLong":"3594120000172545465"}}},{"$group":{"_id":{"URLHash":"$URLHash","EventDate":"$EventDate"},"pageViews":{"$sum":1}}},{"$sort":{"pageViews":-1}},{"$skip":100},{"$limit":10}] +[{"$match":{"CounterID":62,"EventDate":{"$gte":{"$date":"2013-07-01T00:00:00Z"},"$lte":{"$date":"2013-07-31T00:00:00Z"}},"IsRefresh":0,"DontCountHits":0,"URLHash":{"$numberLong":"2868770270353813622"}}},{"$group":{"_id":{"WindowClientWidth":"$WindowClientWidth","WindowClientHeight":"$WindowClientHeight"},"pageViews":{"$sum":1}}},{"$sort":{"pageViews":-1}},{"$skip":10000},{"$limit":10}] +[{"$match":{"CounterID":62,"EventDate":{"$gte":{"$date":"2013-07-14T00:00:00Z"},"$lte":{"$date":"2013-07-15T00:00:00Z"}},"IsRefresh":0,"DontCountHits":0}},{"$group":{"_id":{"$dateTrunc":{"date":"$EventTime","unit":"minute"}},"pageViews":{"$sum":1}}},{"$sort":{"_id":1}},{"$skip":1000},{"$limit":10}]