perf: Delayed writing to database

- Implement delayed writing feature to database for improved performance and reduced write frequency - Update docs
2024-11-27 17:00:51 +01:00 · 2023-03-06 18:57:41 +08:00 · 2023-03-06 18:57:41 +08:00 · c3f18f8b34
commit c3f18f8b34
parent c3cf12769c
8 changed files with 108 additions and 47 deletions
--- a/.gitignore
+++ b/.gitignore
@ -1,5 +1,6 @@
 # See https://help.github.com/articles/ignoring-files/ for more about ignoring files.

+count.db
 /.history

 # dependencies
--- a/Readme.md
+++ b/Readme.md
@ -1,8 +1,8 @@
-# Moe-counter
+# Moe-Counter

 多种风格可选的萌萌计数器

-![Moe-counter](https://count.getloli.com/get/@Moe-counter.github)
+![Moe-Counter](https://count.getloli.com/get/@Moe-counter.github)

 <details>
 <summary>More theme</summary>
@ -26,17 +26,17 @@

 ### Install

-#### Run on Repl.it
+#### Run on Replit

- Open the url [https://repl.it/@journeyad/Moe-counter](https://repl.it/@journeyad/Moe-counter)
+- Open the url [https://replit.com/@journeyad/Moe-Counter](https://replit.com/@journeyad/Moe-Counter)
 - Just hit the **Fork** button
 - And hit the **Run** button

 #### Deploying on your own server

 ```shell
-$ git clone https://github.com/journey-ad/Moe-counter.git
-$ cd Moe-counter
+$ git clone https://github.com/journey-ad/Moe-Counter.git
+$ cd Moe-Counter
 $ yarn install

 $ yarn start
@ -51,7 +51,7 @@ app:
  port: 3000

 db:
-  type: mongodb # sqlite or mongodb
+  type: sqlite # sqlite or mongodb
 ```

 If you use mongodb, you need to specify the environment variable `DB_URL`
@ -61,7 +61,7 @@ If you use mongodb, you need to specify the environment variable `DB_URL`
 export DB_URL=mongodb+srv://account:passwd@***.***.***.mongodb.net/db_count
 ```

-repl.it can use `.env` file, [documentation](https://docs.repl.it/repls/secret-keys)
+replit can use Secrets, [documentation](https://docs.replit.com/programming-ide/storing-sensitive-information-environment-variables)

 ```
 DB_URL="mongodb+srv://account:passwd@***.***.***.mongodb.net/db_count"
@ -69,8 +69,8 @@ DB_URL="mongodb+srv://account:passwd@***.***.***.mongodb.net/db_count"

 ## Credits

-*   [repl.it](https://repl.it/)
-*   [A-SOUL](https://www.asoulworld.com/) <sup>(非官方导航站)</sup>
+*   [replit](https://replit.com/)
+*   [A-SOUL_Official](https://space.bilibili.com/703007996)
 *   [moebooru](https://github.com/moebooru/moebooru)
 *   rule34.xxx NSFW
 *   gelbooru.com NSFW
@ -78,4 +78,4 @@ DB_URL="mongodb+srv://account:passwd@***.***.***.mongodb.net/db_count"

 ## License

-[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fjourney-ad%2FMoe-counter.svg?type=large)](https://app.fossa.com/projects/git%2Bgithub.com%2Fjourney-ad%2FMoe-counter?ref=badge_large)
+[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fjourney-ad%2FMoe-Counter.svg?type=large)](https://app.fossa.com/projects/git%2Bgithub.com%2Fjourney-ad%2FMoe-Counter?ref=badge_large)
--- a/config.yml
+++ b/config.yml
@ -2,4 +2,4 @@ app:
  port: 3000

 db:
-  type: mongodb # sqlite or mongodb
+  type: sqlite # sqlite or mongodb
--- a/db/mongodb.js
+++ b/db/mongodb.js
@ -32,8 +32,24 @@ function setNum(name, num) {
          .exec()
 }

+function setNumMulti(counters) {
+  const bulkOps = counters.map(obj => {
+    const { name, num } = obj
+    return {
+      updateOne: {
+        filter: { name },
+        update: { name, num },
+        upsert: true
+      }
+    }
+  })
+
+  return Count.bulkWrite(bulkOps, { ordered : false })
+}
+
 module.exports = {
  getNum,
  getAll,
-  setNum
+  setNum,
+  setNumMulti
 }
--- a/db/sqlite.js
+++ b/db/sqlite.js
@ -1,11 +1,11 @@
 'use strict'

 const path = require('path')
-const sqlite3 = require('sqlite3')
+const Database = require('better-sqlite3')

-const db = new sqlite3.Database(path.resolve(__dirname, '../count.db'))
+const db = new Database(path.resolve(__dirname, '../count.db'))

-db.run(`CREATE TABLE IF NOT EXISTS tb_count (
+db.exec(`CREATE TABLE IF NOT EXISTS tb_count (
    id    INTEGER      PRIMARY KEY AUTOINCREMENT
                       NOT NULL
                       UNIQUE,
@ -17,44 +17,53 @@ db.run(`CREATE TABLE IF NOT EXISTS tb_count (

 function getNum(name) {
  return new Promise((resolve, reject) => {
-    db.get('SELECT `name`, `num` from tb_count WHERE `name` = ?', name, (err, row) => {
-      if (err) reject(err)
-
-      resolve(row || { name, num: 0 })
-    })
+    const stmt = db.prepare('SELECT `name`, `num` from tb_count WHERE `name` = ?')
+    const row = stmt.get(name)
+    resolve(row || { name, num: 0 })
  })
 }

 function getAll(name) {
  return new Promise((resolve, reject) => {
-    db.get('SELECT * from tb_count', (err, row) => {
-      if (err) reject(err)
-
-      resolve(row)
-    })
+    const stmt = db.prepare('SELECT * from tb_count')
+    const rows = stmt.all()
+    resolve(rows)
  })
 }

 function setNum(name, num) {
  return new Promise((resolve, reject) => {
-    db.run(`INSERT INTO tb_count(\`name\`, \`num\`)
+    db.exec(`INSERT INTO tb_count(\`name\`, \`num\`)
            VALUES($name, $num)
            ON CONFLICT(name) DO
            UPDATE SET \`num\` = $num;`
-      , {
-        $name: name,
-        $num: num
-      }
-      , (err, row) => {
-        if (err) reject(err)
+      ,
+      { $name: name, $num: num }
+    )

-        resolve(row)
-      })
+    resolve()
+  })
+}
+
+function setNumMulti(counters) {
+  return new Promise((resolve, reject) => {
+    const stmt = db.prepare(`INSERT INTO tb_count(\`name\`, \`num\`)
+    VALUES($name, $num)
+    ON CONFLICT(name) DO
+    UPDATE SET \`num\` = $num;`)
+
+    const setMany = db.transaction((counters) => {
+      for (const counter of counters) stmt.run(counter)
+    })
+
+    setMany(counters)
+    resolve()
  })
 }

 module.exports = {
  getNum,
  getAll,
-  setNum
-}
+  setNum,
+  setNumMulti
+}
--- a/index.js
+++ b/index.js
@ -45,7 +45,7 @@ app.get('/get/@:name', async (req, res) => {
  const renderSvg = themify.getCountImage({ count: data.num, theme, length })
  res.send(renderSvg)

-  console.log(data, `theme: ${theme}`)
+  console.log(data, `theme: ${theme}`, `ref: ${req.get('Referrer') || null}`, `ua: ${req.get('User-Agent') || null}`)
 })

 // JSON record
@ -70,16 +70,51 @@ const listener = app.listen(config.app.port || 3000, () => {
  console.log('Your app is listening on port ' + listener.address().port)
 })

+let __cache_counter = {}, shouldPush = false
+
+setInterval(() => {
+  shouldPush = true
+}, 1000 * 60);
+
+async function pushDB() {
+  if (!shouldPush) return
+
+  try {
+    shouldPush = false
+    if (Object.keys(__cache_counter).length === 0) return
+
+    console.log("pushDB", __cache_counter)
+
+    const counters = Object.keys(__cache_counter).map(key => {
+      return {
+        name: key,
+        num: __cache_counter[key]
+      }
+    })
+
+    await db.setNumMulti(counters)
+    __cache_counter = {}
+  } catch (error) {
+    console.log("pushDB is error: ", error)
+  }
+}
+
 async function getCountByName(name) {
  const defaultCount = { name, num: 0 }

  if (name === 'demo') return { name, num: '0123456789' }

  try {
-    const counter = await db.getNum(name) || defaultCount
-    const num = counter.num + 1
-    db.setNum(counter.name, num)
-    return counter
+    if (!(name in __cache_counter)) {
+      const counter = await db.getNum(name) || defaultCount
+      __cache_counter[name] = counter.num + 1
+    } else {
+      __cache_counter[name]++
+    }
+
+    pushDB()
+
+    return { name, num: __cache_counter[name] }

  } catch (error) {
    console.log("get count by name is error: ", error)
--- a/package.json
+++ b/package.json
@ -1,5 +1,5 @@
 {
-  "name": "kawaii-counter",
+  "name": "moe-counter",
  "version": "1.0.0",
  "description": "",
  "main": "index.js",
@ -11,13 +11,13 @@
  "author": "journey-ad",
  "license": "MIT",
  "dependencies": {
+    "better-sqlite3": "^8.1.0",
    "compression": "^1.7.4",
    "config-yml": "^0.10.3",
    "express": "^4.17.1",
    "image-size": "^0.8.3",
    "mime-types": "^2.1.27",
    "mongoose": "^5.9.28",
-    "pug": "^3.0.0",
-    "sqlite3": "^5.0.0"
+    "pug": "^3.0.0"
  }
 }
--- a/views/index.pug
+++ b/views/index.pug
@ -62,7 +62,7 @@ html
      li
        a(href='https://repl.it/', target='_blank', rel='nofollow') repl.it
      li
-        a(href='https://www.asoulworld.com/', target='_blank', title='A-SOUL导航站(非官方)') A-SOUL
+        a(href='https://space.bilibili.com/703007996', target='_blank', title='A-SOUL_Official') A-SOUL
      li
        a(href='https://github.com/moebooru/moebooru', target='_blank', rel='nofollow') moebooru
      li
@ -109,4 +109,4 @@ html
    iframe(src="https://chat.getloli.com/room/@Moe-counter?title=%E8%90%8C%E8%90%8C%E8%AE%A1%E6%95%B0%E5%99%A8%E7%9A%84%E7%95%99%E8%A8%80%E6%9D%BF", scrolling="no", frameborder="0", height="70%", width="26%", style="position: fixed;top: 2%;right: 5%;")

    p.copy
-      a(href='https://github.com/journey-ad/Moe-counter', target='_blank', onclick='_evt_push("click", "normal", "go_github")') source code
+      a(href='https://github.com/journey-ad/Moe-Counter', target='_blank', onclick='_evt_push("click", "normal", "go_github")') source code