From 2c175760a2c903445e89a975439deea755bd34cf Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 31 Jul 2024 19:16:47 +0300 Subject: [PATCH] Overcome "SQLSTATE[HY000]: General error: 1 too many SQL variables" error during merging --- CHANGELOG.md | 1 + composer.json | 1 + composer.lock | 5 +-- .../RevisionLog/Plugin/AbstractPlugin.php | 33 +++++++++++++++++++ .../DatabaseCollectorPlugin/BugsPlugin.php | 2 +- .../ProjectsPlugin.php | 2 +- .../DatabaseCollectorPlugin/RefsPlugin.php | 2 +- .../MergesPlugin.php | 2 +- .../RepositoryCollectorPlugin/PathsPlugin.php | 2 +- .../SummaryPlugin.php | 2 +- 10 files changed, 44 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d01e79c..0810429 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ This project adheres to [Semantic Versioning](http://semver.org/). - The cache delete/set commands were displayed as "cache hit"/"cache miss" operations during verbose output. - When there was a cache miss, then explain why (absent, invalidated, expired) during verbose output. - Queue SVN-Buddy new repository commit discovery, after a new commit in SVN-Buddy was made. +- The `log` and `merge` commands no longer fails with large (>999) revision lists on SQLite <= 3.32.0. ## [0.7.0] - 2024-04-12 ### Added diff --git a/composer.json b/composer.json index 067ef8f..d0d2193 100644 --- a/composer.json +++ b/composer.json @@ -21,6 +21,7 @@ "ext-zlib": "*", "ext-json": "*", "ext-pdo": "*", + "ext-sqlite3": "*", "fiasco/symfony-console-style-markdown": "^1.2" }, "require-dev": { diff --git a/composer.lock b/composer.lock index 41cd4cd..2ac8d07 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "3957a3abbf78d9184b8dcfc307f9ad37", + "content-hash": "3c9f347989c9271d74afec4fecf0f0d5", "packages": [ { "name": "aura/sql", @@ -2619,7 +2619,8 @@ "ext-simplexml": "*", "ext-zlib": "*", "ext-json": "*", - "ext-pdo": "*" + "ext-pdo": "*", + "ext-sqlite3": "*" }, "platform-dev": [], "plugin-api-version": "2.2.0" diff --git a/src/SVNBuddy/Repository/RevisionLog/Plugin/AbstractPlugin.php b/src/SVNBuddy/Repository/RevisionLog/Plugin/AbstractPlugin.php index 91b2ecb..938a60e 100644 --- a/src/SVNBuddy/Repository/RevisionLog/Plugin/AbstractPlugin.php +++ b/src/SVNBuddy/Repository/RevisionLog/Plugin/AbstractPlugin.php @@ -74,6 +74,39 @@ private function _resetParsingStatistics() } } + /** + * Returns raw information about revisions. + * + * @param string $sql SQL. + * @param string $revisions_param_name Revisions param name. + * @param array $revisions Revisions. + * @param boolean $assoc Use first column as returned array key. + * + * @return array + */ + protected function getRawRevisionsData($sql, $revisions_param_name, array $revisions, $assoc = false) + { + // Several queries on a smaller revision set appear to be much faster, then one query for all revisions. + $revisions_data = array(); + + if ( $assoc ) { + foreach ( array_chunk($revisions, 500) as $revisions_chunk ) { + $revisions_data += $this->database->fetchAssoc($sql, array($revisions_param_name => $revisions_chunk)); + } + + return $revisions_data; + } + + foreach ( array_chunk($revisions, 500) as $revisions_chunk ) { + $revisions_data = array_merge( + $revisions_data, + $this->database->fetchAll($sql, array($revisions_param_name => $revisions_chunk)) + ); + } + + return $revisions_data; + } + /** * Hook, that is called before "RevisionLog::refresh" method call. * diff --git a/src/SVNBuddy/Repository/RevisionLog/Plugin/DatabaseCollectorPlugin/BugsPlugin.php b/src/SVNBuddy/Repository/RevisionLog/Plugin/DatabaseCollectorPlugin/BugsPlugin.php index 80f2bdb..572b5d6 100644 --- a/src/SVNBuddy/Repository/RevisionLog/Plugin/DatabaseCollectorPlugin/BugsPlugin.php +++ b/src/SVNBuddy/Repository/RevisionLog/Plugin/DatabaseCollectorPlugin/BugsPlugin.php @@ -436,7 +436,7 @@ public function getRevisionsData(array $revisions) $sql = 'SELECT Revision, Bug FROM CommitBugs WHERE Revision IN (:revisions)'; - $revisions_data = $this->database->fetchAll($sql, array('revisions' => $revisions)); + $revisions_data = $this->getRawRevisionsData($sql, 'revisions', $revisions); foreach ( $revisions_data as $revision_data ) { $revision = $revision_data['Revision']; diff --git a/src/SVNBuddy/Repository/RevisionLog/Plugin/DatabaseCollectorPlugin/ProjectsPlugin.php b/src/SVNBuddy/Repository/RevisionLog/Plugin/DatabaseCollectorPlugin/ProjectsPlugin.php index d3bf448..ddc8611 100644 --- a/src/SVNBuddy/Repository/RevisionLog/Plugin/DatabaseCollectorPlugin/ProjectsPlugin.php +++ b/src/SVNBuddy/Repository/RevisionLog/Plugin/DatabaseCollectorPlugin/ProjectsPlugin.php @@ -161,7 +161,7 @@ public function getRevisionsData(array $revisions) $sql = 'SELECT Revision, ProjectId FROM CommitProjects WHERE Revision IN (:revisions)'; - $revisions_data = $this->database->fetchAll($sql, array('revisions' => $revisions)); + $revisions_data = $this->getRawRevisionsData($sql, 'revisions', $revisions); foreach ( $revisions_data as $revision_data ) { $revision = $revision_data['Revision']; diff --git a/src/SVNBuddy/Repository/RevisionLog/Plugin/DatabaseCollectorPlugin/RefsPlugin.php b/src/SVNBuddy/Repository/RevisionLog/Plugin/DatabaseCollectorPlugin/RefsPlugin.php index d360d0b..10a0618 100644 --- a/src/SVNBuddy/Repository/RevisionLog/Plugin/DatabaseCollectorPlugin/RefsPlugin.php +++ b/src/SVNBuddy/Repository/RevisionLog/Plugin/DatabaseCollectorPlugin/RefsPlugin.php @@ -151,7 +151,7 @@ public function getRevisionsData(array $revisions) FROM CommitRefs cr JOIN ProjectRefs pr ON pr.Id = cr.RefId WHERE cr.Revision IN (:revisions)'; - $revisions_data = $this->database->fetchAll($sql, array('revisions' => $revisions)); + $revisions_data = $this->getRawRevisionsData($sql, 'revisions', $revisions); foreach ( $revisions_data as $revision_data ) { $revision = $revision_data['Revision']; diff --git a/src/SVNBuddy/Repository/RevisionLog/Plugin/RepositoryCollectorPlugin/MergesPlugin.php b/src/SVNBuddy/Repository/RevisionLog/Plugin/RepositoryCollectorPlugin/MergesPlugin.php index 8df71a3..dce7bc5 100644 --- a/src/SVNBuddy/Repository/RevisionLog/Plugin/RepositoryCollectorPlugin/MergesPlugin.php +++ b/src/SVNBuddy/Repository/RevisionLog/Plugin/RepositoryCollectorPlugin/MergesPlugin.php @@ -165,7 +165,7 @@ public function getRevisionsData(array $revisions) $sql = 'SELECT MergeRevision, MergedRevision FROM Merges WHERE MergedRevision IN (:merged_revisions)'; - $revisions_data = $this->database->fetchAll($sql, array('merged_revisions' => $revisions)); + $revisions_data = $this->getRawRevisionsData($sql, 'merged_revisions', $revisions); foreach ( $revisions_data as $revision_data ) { $merge_revision = $revision_data['MergeRevision']; diff --git a/src/SVNBuddy/Repository/RevisionLog/Plugin/RepositoryCollectorPlugin/PathsPlugin.php b/src/SVNBuddy/Repository/RevisionLog/Plugin/RepositoryCollectorPlugin/PathsPlugin.php index 80b0d86..139ca3a 100644 --- a/src/SVNBuddy/Repository/RevisionLog/Plugin/RepositoryCollectorPlugin/PathsPlugin.php +++ b/src/SVNBuddy/Repository/RevisionLog/Plugin/RepositoryCollectorPlugin/PathsPlugin.php @@ -928,7 +928,7 @@ public function getRevisionsData(array $revisions) JOIN Paths p1 ON p1.Id = cp.PathId LEFT JOIN Paths p2 ON p2.Id = cp.CopyPathId WHERE cp.Revision IN (:revision_ids)'; - $revisions_data = $this->database->fetchAll($sql, array('revision_ids' => $revisions)); + $revisions_data = $this->getRawRevisionsData($sql, 'revision_ids', $revisions); foreach ( $revisions_data as $revision_data ) { $revision = $revision_data['Revision']; diff --git a/src/SVNBuddy/Repository/RevisionLog/Plugin/RepositoryCollectorPlugin/SummaryPlugin.php b/src/SVNBuddy/Repository/RevisionLog/Plugin/RepositoryCollectorPlugin/SummaryPlugin.php index fe05b72..d03810c 100644 --- a/src/SVNBuddy/Repository/RevisionLog/Plugin/RepositoryCollectorPlugin/SummaryPlugin.php +++ b/src/SVNBuddy/Repository/RevisionLog/Plugin/RepositoryCollectorPlugin/SummaryPlugin.php @@ -138,7 +138,7 @@ public function getRevisionsData(array $revisions) $sql = 'SELECT Revision, Author AS author, Date AS date, Message AS msg FROM Commits WHERE Revision IN (:revision_ids)'; - $results = $this->database->fetchAssoc($sql, array('revision_ids' => $revisions)); + $results = $this->getRawRevisionsData($sql, 'revision_ids', $revisions, true); foreach ( array_keys($results) as $revision ) { unset($results[$revision]['Revision']);