Skip to content

Commit

Permalink
πŸ”οΈ perf: add sitemap and robot.txt to improve SEO (lobehub#1337)
Browse files Browse the repository at this point in the history
* πŸ”οΈ fix: add sitemap and robots.txt

* πŸ› fix: fix canonical url

* πŸ› fix: fix canonical url

* πŸ› fix: fix canonical url

* πŸ› fix: fix deploy url

* πŸ› fix: fix canonical url

* πŸ› fix: fix the robots config
  • Loading branch information
arvinxx authored and 573dave committed Mar 5, 2024
1 parent 8b02353 commit d0a37b4
Show file tree
Hide file tree
Showing 11 changed files with 99 additions and 5 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -56,3 +56,4 @@ next-env.d.ts
.env
public/*.js
bun.lockb
sitemap*.xml
3 changes: 3 additions & 0 deletions .npmrc
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
lockfile=false
resolution-mode=highest

enable-pre-post-scripts=true

public-hoist-pattern[]=*@umijs/lint*
public-hoist-pattern[]=*changelog*
public-hoist-pattern[]=*commitlint*
Expand Down
54 changes: 54 additions & 0 deletions next-sitemap.config.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
import { glob } from 'glob';

const isVercelPreview = process.env.VERCEL === '1' && process.env.VERCEL_ENV !== 'production';

const prodUrl = process.env.SITE_URL || 'https://chat-preview.lobehub.com';

const vercelPreviewUrl = `https://${process.env.VERCEL_URL}`;

const siteUrl = isVercelPreview ? vercelPreviewUrl : prodUrl;

/** @type {import('next-sitemap').IConfig} */
const config = {
// next-sitemap does not work with app dir inside the /src dir (and have other problems e.g. with route groups)
// https://github.com/iamvishnusankar/next-sitemap/issues/700#issuecomment-1759458127
// https://github.com/iamvishnusankar/next-sitemap/issues/701
// additionalPaths is a workaround for this (once the issues are fixed, we can remove it)
additionalPaths: async () => {
const routes = await glob('src/app/**/page.{md,mdx,ts,tsx}', {
cwd: new URL('.', import.meta.url).pathname,
});

// https://nextjs.org/docs/app/building-your-application/routing/colocation#private-folders
const publicRoutes = routes.filter(
(page) => !page.split('/').some((folder) => folder.startsWith('_')),
);

// https://nextjs.org/docs/app/building-your-application/routing/colocation#route-groups
const publicRoutesWithoutRouteGroups = publicRoutes.map((page) =>
page
.split('/')
.filter((folder) => !folder.startsWith('(') && !folder.endsWith(')'))
.join('/'),
);

const locs = publicRoutesWithoutRouteGroups.map((route) => {
const path = route.replace(/^src\/app/, '').replace(/\/[^/]+$/, '');
const loc = path === '' ? siteUrl : `${siteUrl}/${path}`;

return loc;
});

const paths = locs.map((loc) => ({
changefreq: 'daily',
lastmod: new Date().toISOString(),
loc,
priority: 0.7,
}));

return paths;
},
siteUrl,
};

export default config;
2 changes: 2 additions & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
"sideEffects": false,
"scripts": {
"build": "next build",
"postbuild": "next-sitemap --config next-sitemap.config.mjs",
"build:analyze": "ANALYZE=true next build",
"build:docker": "DOCKER=true next build",
"dev": "next dev -p 3010",
Expand Down Expand Up @@ -107,6 +108,7 @@
"nanoid": "^5",
"next": "^14.1",
"next-auth": "5.0.0-beta.11",
"next-sitemap": "^4.2.3",
"numeral": "^2.0.6",
"nuqs": "^1.15.4",
"openai": "^4.22",
Expand Down
7 changes: 7 additions & 0 deletions public/robots.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
Sitemap: /sitemap.xml

User-agent: *
Allow: /$
Allow: /welcome
Allow: /market*
Disallow: /
7 changes: 7 additions & 0 deletions src/app/market/page.tsx
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
import { Metadata } from 'next';

import { getCanonicalUrl } from '@/const/url';
import { isMobileDevice } from '@/utils/responsive';

import DesktopPage from './(desktop)';
Expand All @@ -10,3 +13,7 @@ export default () => {

return <Page />;
};

export const metadata: Metadata = {
alternates: { canonical: getCanonicalUrl('/market') },
};
7 changes: 4 additions & 3 deletions src/app/metadata.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,14 @@ import { Metadata } from 'next';

import { getClientConfig } from '@/config/client';
import { getServerConfig } from '@/config/server';
import { OFFICIAL_URL } from '@/const/url';

import pkg from '../../package.json';

const title = 'LobeChat';
const { description, homepage } = pkg;

const { METADATA_BASE_URL = 'https://chat-preview.lobehub.com/' } = getServerConfig();
const { SITE_URL = OFFICIAL_URL } = getServerConfig();
const { BASE_PATH } = getClientConfig();

// if there is a base path, then we don't need the manifest
Expand All @@ -28,7 +29,7 @@ const metadata: Metadata = {
'https://registry.npmmirror.com/@lobehub/assets-favicons/latest/files/assets/favicon.ico',
},
manifest: noManifest ? undefined : '/manifest.json',
metadataBase: new URL(METADATA_BASE_URL),
metadataBase: new URL(SITE_URL),
openGraph: {
description: description,
images: [
Expand Down Expand Up @@ -58,11 +59,11 @@ const metadata: Metadata = {
},
twitter: {
card: 'summary_large_image',
creator: '@lobehub',
description,
images: [
'https://registry.npmmirror.com/@lobehub/assets-favicons/latest/files/assets/og-960x540.png',
],
site: '@lobehub',
title,
},
};
Expand Down
8 changes: 8 additions & 0 deletions src/app/page.tsx
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
import { Metadata } from 'next';

import { getCanonicalUrl } from '@/const/url';

import Page from './home';
import Redirect from './home/Redirect';

Expand All @@ -9,3 +13,7 @@ const Index = () => (
);

export default Index;

export const metadata: Metadata = {
alternates: { canonical: getCanonicalUrl('/') },
};
7 changes: 7 additions & 0 deletions src/app/welcome/page.tsx
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
import { Metadata } from 'next';

import { getCanonicalUrl } from '@/const/url';
import { isMobileDevice } from '@/utils/responsive';

import DesktopPage from './(desktop)';
Expand All @@ -12,3 +15,7 @@ const Page = () => {
};

export default Page;

export const metadata: Metadata = {
alternates: { canonical: getCanonicalUrl('/welcome') },
};
4 changes: 2 additions & 2 deletions src/config/server/app.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ declare global {

IMGUR_CLIENT_ID?: string;

METADATA_BASE_URL?: string;
SITE_URL?: string;

AGENTS_INDEX_URL?: string;

Expand Down Expand Up @@ -38,7 +38,7 @@ export const getAppConfig = () => {

SHOW_ACCESS_CODE_CONFIG: !!ACCESS_CODES.length,

METADATA_BASE_URL: process.env.METADATA_BASE_URL,
SITE_URL: process.env.SITE_URL,

IMGUR_CLIENT_ID: process.env.IMGUR_CLIENT_ID || DEFAULT_IMAGUR_CLIENT_ID,

Expand Down
4 changes: 4 additions & 0 deletions src/const/url.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@ import { withBasePath } from '@/utils/basePath';
import pkg from '../../package.json';
import { INBOX_SESSION_ID } from './session';

export const OFFICIAL_URL = 'https://chat-preview.lobehub.com/';

export const getCanonicalUrl = (path: string) => urlJoin(OFFICIAL_URL, path);

export const GITHUB = pkg.homepage;
export const CHANGELOG = urlJoin(GITHUB, 'blob/master/CHANGELOG.md');

Expand Down

0 comments on commit d0a37b4

Please sign in to comment.