retry wait on db exceptions

1 year ago · 109cebc0c9
5 changed files with 62 additions and 27 deletions
--- a/conan.lock
+++ b/conan.lock
@ -19,7 +19,7 @@
				@@ -19,7 +19,7 @@
   "1": {
    "ref": "fmt/9.1.0",
    "options": "fPIC=True\nheader_only=False\nshared=False\nwith_fmt_alias=False\nwith_os_api=True",
-    "package_id": "e0e003fc38e6dcc03cdfbaf52057d71baa23adad",
+    "package_id": "00f13467ba683328f2424359bddcf6e502820f17",
    "prev": "0",
    "context": "host"
   },
@ -33,15 +33,16 @@
				@@ -33,15 +33,16 @@
   "3": {
    "ref": "tree-sitter/0.20.6",
    "options": "fPIC=True\nshared=False",
-    "package_id": "d13c97721d7bdc192ca6529684f2b79beeae8a7c",
+    "package_id": "2a19826344ff00be1c04403f2f8e7008ed3a7cc6",
    "prev": "0",
    "context": "host"
   },
   "4": {
    "ref": "sqlitecpp/3.2.1",
    "options": "fPIC=True\nshared=False\nstack_protection=True\nsqlite3:build_executable=True\nsqlite3:disable_gethostuuid=False\nsqlite3:enable_column_metadata=True\nsqlite3:enable_dbpage_vtab=False\nsqlite3:enable_dbstat_vtab=False\nsqlite3:enable_default_secure_delete=False\nsqlite3:enable_default_vfs=True\nsqlite3:enable_explain_comments=False\nsqlite3:enable_fts3=False\nsqlite3:enable_fts3_parenthesis=False\nsqlite3:enable_fts4=False\nsqlite3:enable_fts5=False\nsqlite3:enable_json1=False\nsqlite3:enable_math_functions=True\nsqlite3:enable_preupdate_hook=False\nsqlite3:enable_rtree=True\nsqlite3:enable_soundex=False\nsqlite3:enable_unlock_notify=True\nsqlite3:fPIC=True\nsqlite3:max_blob_size=None\nsqlite3:max_column=None\nsqlite3:max_variable_number=None\nsqlite3:omit_deprecated=False\nsqlite3:omit_load_extension=False\nsqlite3:shared=False\nsqlite3:threadsafe=1\nsqlite3:use_alloca=False",
-    "package_id": "07edb47a886c224dfcbe7d7df487e4e73aa2ee6e",
+    "package_id": "00f6be3a0c8e9987d6244aaf78c34ef2142e1e84",
    "prev": "0",
+    "modified": true,
    "requires": [
     "5"
    ],
@ -50,15 +51,17 @@
				@@ -50,15 +51,17 @@
   "5": {
    "ref": "sqlite3/3.40.0",
    "options": "build_executable=True\ndisable_gethostuuid=False\nenable_column_metadata=True\nenable_dbpage_vtab=False\nenable_dbstat_vtab=False\nenable_default_secure_delete=False\nenable_default_vfs=True\nenable_explain_comments=False\nenable_fts3=False\nenable_fts3_parenthesis=False\nenable_fts4=False\nenable_fts5=False\nenable_json1=False\nenable_math_functions=True\nenable_preupdate_hook=False\nenable_rtree=True\nenable_soundex=False\nenable_unlock_notify=True\nfPIC=True\nmax_blob_size=None\nmax_column=None\nmax_variable_number=None\nomit_deprecated=False\nomit_load_extension=False\nshared=False\nthreadsafe=1\nuse_alloca=False",
-    "package_id": "423c3ee42011b37f8a7a6a2c28aa0edfab60ff4d",
+    "package_id": "668a23a956a1ae835b578205ecc8274cfd29ae40",
    "prev": "0",
+    "modified": true,
    "context": "host"
   },
   "6": {
    "ref": "libgit2/1.5.0",
    "options": "fPIC=True\nshared=False\nthreadsafe=True\nwith_https=openssl\nwith_libssh2=True\nwith_ntlmclient=True\nwith_regex=builtin\nwith_sha1=collisiondetection\nhttp_parser:fPIC=True\nhttp_parser:shared=False\nlibssh2:crypto_backend=openssl\nlibssh2:enable_crypt_none=False\nlibssh2:enable_debug_logging=False\nlibssh2:enable_mac_none=False\nlibssh2:fPIC=True\nlibssh2:shared=False\nlibssh2:with_zlib=True\nopenssl:386=False\nopenssl:enable_weak_ssl_ciphers=False\nopenssl:fPIC=True\nopenssl:no_aria=False\nopenssl:no_asm=False\nopenssl:no_async=False\nopenssl:no_bf=False\nopenssl:no_blake2=False\nopenssl:no_camellia=False\nopenssl:no_cast=False\nopenssl:no_chacha=False\nopenssl:no_cms=False\nopenssl:no_comp=False\nopenssl:no_ct=False\nopenssl:no_deprecated=False\nopenssl:no_des=False\nopenssl:no_dgram=False\nopenssl:no_dh=False\nopenssl:no_dsa=False\nopenssl:no_dso=False\nopenssl:no_ec=False\nopenssl:no_ecdh=False\nopenssl:no_ecdsa=False\nopenssl:no_engine=False\nopenssl:no_filenames=False\nopenssl:no_gost=False\nopenssl:no_hmac=False\nopenssl:no_idea=False\nopenssl:no_md4=False\nopenssl:no_md5=False\nopenssl:no_mdc2=False\nopenssl:no_ocsp=False\nopenssl:no_pinshared=False\nopenssl:no_rc2=False\nopenssl:no_rfc3779=False\nopenssl:no_rmd160=False\nopenssl:no_rsa=False\nopenssl:no_seed=False\nopenssl:no_sha=False\nopenssl:no_sm2=False\nopenssl:no_sm3=False\nopenssl:no_sm4=False\nopenssl:no_sock=False\nopenssl:no_srp=False\nopenssl:no_srtp=False\nopenssl:no_sse2=False\nopenssl:no_ssl=False\nopenssl:no_ssl3=False\nopenssl:no_stdio=False\nopenssl:no_tests=False\nopenssl:no_threads=False\nopenssl:no_tls1=False\nopenssl:no_ts=False\nopenssl:no_whirlpool=False\nopenssl:openssldir=None\nopenssl:shared=False\nzlib:fPIC=True\nzlib:shared=False",
-    "package_id": "6ed20d33bf0d208a8b0717399dc35c204becaa5a",
+    "package_id": "2620bf4535f8ff21786c486a8c0d30e09705e660",
    "prev": "0",
+    "modified": true,
    "requires": [
     "7",
     "8",
@ -70,22 +73,24 @@
				@@ -70,22 +73,24 @@
   "7": {
    "ref": "zlib/1.2.12",
    "options": "fPIC=True\nshared=False",
-    "package_id": "d13c97721d7bdc192ca6529684f2b79beeae8a7c",
+    "package_id": "2a19826344ff00be1c04403f2f8e7008ed3a7cc6",
    "prev": "0",
+    "modified": true,
    "context": "host"
   },
   "8": {
    "ref": "http_parser/2.9.4",
    "options": "fPIC=True\nshared=False",
-    "package_id": "d13c97721d7bdc192ca6529684f2b79beeae8a7c",
+    "package_id": "2a19826344ff00be1c04403f2f8e7008ed3a7cc6",
    "prev": "0",
    "context": "host"
   },
   "9": {
    "ref": "libssh2/1.10.0",
    "options": "crypto_backend=openssl\nenable_crypt_none=False\nenable_debug_logging=False\nenable_mac_none=False\nfPIC=True\nshared=False\nwith_zlib=True\nopenssl:386=False\nopenssl:enable_weak_ssl_ciphers=False\nopenssl:fPIC=True\nopenssl:no_aria=False\nopenssl:no_asm=False\nopenssl:no_async=False\nopenssl:no_bf=False\nopenssl:no_blake2=False\nopenssl:no_camellia=False\nopenssl:no_cast=False\nopenssl:no_chacha=False\nopenssl:no_cms=False\nopenssl:no_comp=False\nopenssl:no_ct=False\nopenssl:no_deprecated=False\nopenssl:no_des=False\nopenssl:no_dgram=False\nopenssl:no_dh=False\nopenssl:no_dsa=False\nopenssl:no_dso=False\nopenssl:no_ec=False\nopenssl:no_ecdh=False\nopenssl:no_ecdsa=False\nopenssl:no_engine=False\nopenssl:no_filenames=False\nopenssl:no_gost=False\nopenssl:no_hmac=False\nopenssl:no_idea=False\nopenssl:no_md4=False\nopenssl:no_md5=False\nopenssl:no_mdc2=False\nopenssl:no_ocsp=False\nopenssl:no_pinshared=False\nopenssl:no_rc2=False\nopenssl:no_rfc3779=False\nopenssl:no_rmd160=False\nopenssl:no_rsa=False\nopenssl:no_seed=False\nopenssl:no_sha=False\nopenssl:no_sm2=False\nopenssl:no_sm3=False\nopenssl:no_sm4=False\nopenssl:no_sock=False\nopenssl:no_srp=False\nopenssl:no_srtp=False\nopenssl:no_sse2=False\nopenssl:no_ssl=False\nopenssl:no_ssl3=False\nopenssl:no_stdio=False\nopenssl:no_tests=False\nopenssl:no_threads=False\nopenssl:no_tls1=False\nopenssl:no_ts=False\nopenssl:no_whirlpool=False\nopenssl:openssldir=None\nopenssl:shared=False\nzlib:fPIC=True\nzlib:shared=False",
-    "package_id": "4760125fd09127af98dbf68387bd44e807573faa",
+    "package_id": "23aa532ba63eddae5cee0f4ecf92be1a96959ae4",
    "prev": "0",
+    "modified": true,
    "requires": [
     "7",
     "10"
@ -95,14 +100,15 @@
				@@ -95,14 +100,15 @@
   "10": {
    "ref": "openssl/1.1.1o",
    "options": "386=False\nenable_weak_ssl_ciphers=False\nfPIC=True\nno_aria=False\nno_asm=False\nno_async=False\nno_bf=False\nno_blake2=False\nno_camellia=False\nno_cast=False\nno_chacha=False\nno_cms=False\nno_comp=False\nno_ct=False\nno_deprecated=False\nno_des=False\nno_dgram=False\nno_dh=False\nno_dsa=False\nno_dso=False\nno_ec=False\nno_ecdh=False\nno_ecdsa=False\nno_engine=False\nno_filenames=False\nno_gost=False\nno_hmac=False\nno_idea=False\nno_md4=False\nno_md5=False\nno_mdc2=False\nno_ocsp=False\nno_pinshared=False\nno_rc2=False\nno_rfc3779=False\nno_rmd160=False\nno_rsa=False\nno_seed=False\nno_sha=False\nno_sm2=False\nno_sm3=False\nno_sm4=False\nno_sock=False\nno_srp=False\nno_srtp=False\nno_sse2=False\nno_ssl=False\nno_ssl3=False\nno_stdio=False\nno_tests=False\nno_threads=False\nno_tls1=False\nno_ts=False\nno_whirlpool=False\nopenssldir=None\nshared=False",
-    "package_id": "d13c97721d7bdc192ca6529684f2b79beeae8a7c",
+    "package_id": "2a19826344ff00be1c04403f2f8e7008ed3a7cc6",
    "prev": "0",
+    "modified": true,
    "context": "host"
   },
   "11": {
    "ref": "catch2/3.3.1",
    "options": "console_width=80\ndefault_reporter=None\nfPIC=True\nshared=False\nwith_prefix=False",
-    "package_id": "6b952e07b144105007d6beb6165b6d6b25ff4787",
+    "package_id": "cd1282dfb2c752078c9bbb64f4388c8a4a127008",
    "prev": "0",
    "context": "host"
   }
@ -110,5 +116,5 @@
				@@ -110,5 +116,5 @@
  "revisions_enabled": false
 },
 "version": "0.4",
- "profile_host": "[settings]\narch=x86_64\narch_build=x86_64\nbuild_type=Debug\ncompiler=gcc\ncompiler.libcxx=libstdc++\ncompiler.version=12\nos=Linux\nos_build=Linux\n[options]\n[build_requires]\n[env]\n"
+ "profile_host": "[settings]\narch=x86_64\narch_build=x86_64\nbuild_type=Release\ncompiler=gcc\ncompiler.libcxx=libstdc++\ncompiler.version=12\nos=Linux\nos_build=Linux\n[options]\n[build_requires]\n[env]\n"
 }
--- a/database.db
+++ b/database.db
--- a/source/db.cpp
+++ b/source/db.cpp
@ -6,6 +6,8 @@
				@@ -6,6 +6,8 @@
 #include <fmt/color.h>
 #include <fmt/format.h>
 #include <memory>
+#include <chrono>
+#include <thread>


 #include "db.hpp"
@ -18,7 +20,7 @@ class sqlite_handle : public db_handle {
				@@ -18,7 +20,7 @@ class sqlite_handle : public db_handle {
    sqlite_handle(const std::string &filename) :
        database(filename, SQLite::OPEN_READWRITE|SQLite::OPEN_CREATE) {}

-    void store(std::vector<source_file> records) final;
+    bool store(std::vector<source_file> records) final;
    void init() final;
 };

@ -44,7 +46,7 @@ static const std::string insert_function_query = "INSERT INTO function_definitio
				@@ -44,7 +46,7 @@ static const std::string insert_function_query = "INSERT INTO function_definitio



-void insert_function_record(const function &r, SQLite::Database &db) {
+int insert_function_record(const function &r, SQLite::Database &db) {

    SQLite::Statement query {db, insert_function_query};
    int rec = 1;
@ -54,14 +56,15 @@ void insert_function_record(const function &r, SQLite::Database &db) {
				@@ -54,14 +56,15 @@ void insert_function_record(const function &r, SQLite::Database &db) {
    query.bind(rec++, r.start_line);
    query.bind(rec++, r.length_lines);
    try {
-        query.exec();
+        return query.exec();
    } catch (SQLite::Exception &ex) {
        fmt::print(fmt::fg(fmt::color::red), "SQLite Exception {}: {}\n", ex.getErrorCode(), ex.getErrorStr());
+        return 0;
    }
 }


-void insert_record(const source_file &r, SQLite::Database &db) {
+int insert_record(const source_file &r, SQLite::Database &db) {

    SQLite::Statement query {db, insert_query};
    int rec = 1;
@ -76,9 +79,10 @@ void insert_record(const source_file &r, SQLite::Database &db) {
				@@ -76,9 +79,10 @@ void insert_record(const source_file &r, SQLite::Database &db) {
    query.bind(rec++, r.function_defs);
    query.bind(rec++, r.function_lines);
    try {
-        query.exec();
+        return query.exec();
    } catch (SQLite::Exception &ex) {
        fmt::print(fmt::fg(fmt::color::red), "SQLite Exception {}: {}\n", ex.getErrorCode(), ex.getErrorStr());
+        return 0;
    }
 }

@ -89,27 +93,43 @@ std::shared_ptr<db_handle> open_database(const std::string &filename) {
				@@ -89,27 +93,43 @@ std::shared_ptr<db_handle> open_database(const std::string &filename) {


 void sqlite_handle::init() {
-    database.exec(query_init);
+    while (database.tryExec(query_init) != 0) {
+        std::this_thread::sleep_for(std::chrono::milliseconds(300));
+    }
 }


-void db_handle::store(std::vector<source_file> records) {
+bool db_handle::store(std::vector<source_file> records) {
    throw std::runtime_error("Not implemented");
 }

-void sqlite_handle::store(std::vector<source_file> records) {
+bool sqlite_handle::store(std::vector<source_file> records) {
    fmt::print("Storing {} records.\n",records.size());
+
+
    SQLite::Transaction trans {database};
+    bool failed = false; 
    for (const auto& r : records) {
-        insert_record(r, database);
+        if (0 == insert_record(r, database)) {
+            failed = true;
+        }

        if (r.function_info) {
            for (const auto &func : r.function_info->functions) {
-                insert_function_record(func, database);
+                if (0 == insert_function_record(func, database)) {
+                    failed = true;
+                }
+                
            }
        }
    }
-    trans.commit();
+    try {
+        trans.commit();
+    } catch (SQLite::Exception &ex) {
+        fmt::print(fmt::fg(fmt::color::red), "SQLite Exception {}: {}\n", ex.getErrorCode(), ex.getErrorStr());
+        return true;
+    }
+    return failed;
 }


--- a/source/db.hpp
+++ b/source/db.hpp
@ -7,7 +7,7 @@
				@@ -7,7 +7,7 @@

 class db_handle {
    public:
-    virtual void store(std::vector<source_file> records) = 0;
+    virtual bool store(std::vector<source_file> records) = 0;
    virtual void init() = 0;
    virtual ~db_handle() = default;
 };
--- a/source/main.cpp
+++ b/source/main.cpp
@ -1,3 +1,4 @@
				@@ -1,3 +1,4 @@
+#include <chrono>
 #include <cstddef>
 #include <filesystem>
 #include <iostream>
@ -7,6 +8,9 @@
				@@ -7,6 +8,9 @@
 #include <string_view>
 #include <optional>
 #include <vector>
+#include <thread>
+
+

 #include <cxxopts.hpp>
 #include <fmt/format.h>
@ -56,11 +60,17 @@ auto main(int argc, char **argv) -> int
				@@ -56,11 +60,17 @@ auto main(int argc, char **argv) -> int
                files.emplace_back(dir_entry.path().string(), parser);
                //fmt::print("{}\n", files.back().as_string());
                if (++count % 500 == 0) {
-                    database->store(files);
-                    files.clear();
+                    if (!database->store(files)) {
+                        files.clear();
+                    }
                }
            }
-            database->store(files);
+
+            while (database->store(files)) {
+                // retry while fail
+                std::this_thread::sleep_for(std::chrono::milliseconds(300));
+            }
+            

        } else  {
            auto s = source_file(path.string());
@ -73,7 +83,6 @@ auto main(int argc, char **argv) -> int
				@@ -73,7 +83,6 @@ auto main(int argc, char **argv) -> int
                std::getline(std::cin, line);
                if (line != "exit") {
                    run_query(s, line);
-                    
                }

            }