Update sources map

Pawel-Szydlo · Aug 7, 2023 · 6c7631f · 6c7631f
1 parent 5947152
commit 6c7631f
Show file tree

Hide file tree

Showing 3 changed files with 26 additions and 17 deletions.
diff --git a/functions/sitemap/index.js b/functions/sitemap/index.js
@@ -12,7 +12,10 @@ export async function onRequest({ request, next, env }) {
         <loc>https://near.social/sitemap/profiles/</loc>
     </sitemap>
     <sitemap>
-        <loc>https://near.social/sitemap/sources/</loc>
+        <loc>https://near.social/sitemap/sources/0</loc>
+    </sitemap>
+    <sitemap>
+        <loc>https://near.social/sitemap/sources/50000</loc>
     </sitemap>
 </sitemapindex>`,
     {

diff --git a/functions/sitemap/posts/[index].js b/functions/sitemap/posts/[index].js
@@ -3,19 +3,16 @@ import { socialIndex } from "../../common";
 const Limit = 50000;
 
 export const generateSitemapPosts = async (env, offset) => {
-  const posts = await socialIndex("post", "main", {
-    from: offset,
-    limit: Limit,
-  });
-  return posts
-    .map(
-      (post) =>
-        `  <url>
+  const posts = await socialIndex("post", "main", {});
+  const urls = posts.map(
+    (post) =>
+      `  <url>
     <loc>https://near.social/mob.near/widget/MainPage.Post.Page?accountId=${post.accountId}&amp;blockHeight=${post.blockHeight}</loc>
     <changefreq>never</changefreq>
   </url>`
-    )
-    .join("\n");
+  );
+  console.log("urls count", urls.length);
+  return urls.slice(offset, offset + Limit).join("\n");
 };
 
 export async function onRequest({ request, env, next }) {

diff --git a/functions/sitemap/sources/index.js → functions/sitemap/sources/[index].js b/functions/sitemap/sources/index.js → functions/sitemap/sources/[index].js
@@ -1,12 +1,13 @@
 import { socialKeys } from "../../common";
 
 const MinBlockHeight = 75942518;
+const LIMIT = 50000;
 
-export const generateSitemapSources = async (env) => {
+export const generateSitemapSources = async (env, offset) => {
   const data = await socialKeys("*/widget/*", null, {
     return_type: "History",
   });
-  return Object.entries(data)
+  const urls = Object.entries(data)
     .map(([accountId, widget]) =>
       Object.entries(widget.widget)
         .map(([widgetId, blockHeights]) =>
@@ -22,15 +23,23 @@ export const generateSitemapSources = async (env) => {
         )
         .flat()
     )
-    .flat()
-    .join("\n");
+    .flat();
+  console.log("urls count", urls.length);
+  return urls.slice(offset, offset + LIMIT).join("\n");
 };
 
-export async function onRequest({ env }) {
+export async function onRequest({ env, request, next }) {
+  const url = new URL(request.url);
+  const parts = url.pathname.split("/");
+  if (parts.length !== 4) {
+    return next();
+  }
+  const offset = parseInt(parts[3]);
+
   return new Response(
     `<?xml version="1.0" encoding="UTF-8"?>
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
-${await generateSitemapSources(env)}
+${await generateSitemapSources(env, offset)}
 </urlset>`,
     {
       headers: {