Add timeout-safe auto schema sync with stop control

app/Config/Routes.php

@@ -227,6 +227,8 @@ $routes->get('setup/db-tools', 'Setup::dbTools');
 $routes->post('setup/db-tools/generate', 'Setup::generateMigrations');
 $routes->post('setup/db-tools/export-master-schema', 'Setup::exportMasterSchema');
 $routes->post('setup/db-tools/schema-sync', 'Setup::schemaSync');
+$routes->post('setup/db-tools/schema-sync-step', 'Setup::schemaSyncStep');
+$routes->post('setup/db-tools/prepare-filesystem', 'Setup::prepareFilesystem');
 $routes->post('setup/db-tools/ensure-admin', 'Setup::ensureAdminLogin');
 $routes->post('setup/db-tools/complete', 'Setup::complete');
 

app/Controllers/Setup.php

@@ -110,31 +110,87 @@ class Setup extends BaseController
         @set_time_limit(0);
         @ignore_user_abort(true);
 
+        $action = strtolower(trim((string) $this->request->getPost('sync_action')));
+        if (!in_array($action, ['analyze', 'apply'], true)) {
+            $action = 'analyze';
+        }
+
+        $this->lastSyncResult = $this->executeSchemaSyncAction($action);
+
+        return view('setup/db_setup', $this->buildDbToolsViewData());
+    }
+
+    public function schemaSyncStep()
+    {
+        if ($deny = $this->ensureSetupAccess()) {
+            return $deny;
+        }
+
+        @ini_set('memory_limit', (string) env('setup.sync.memory_limit', '1024M'));
+        @ini_set('max_execution_time', '0');
+        @set_time_limit(0);
+        @ignore_user_abort(true);
+
+        $result = $this->executeSchemaSyncAction('apply');
+        $this->lastSyncResult = $result;
+
+        $sqlCount = is_array($result['sql'] ?? null) ? count($result['sql']) : 0;
+        $applyErrors = is_array($result['apply_errors'] ?? null) ? count($result['apply_errors']) : 0;
+        $applied = (int) ($result['applied'] ?? 0);
+        $truncated = !empty($result['truncated']) || !empty($result['apply_truncated']);
+
+        $shouldContinue = $applyErrors === 0
+            && ($truncated || ($sqlCount > 0 && $applied > 0));
+
+        return $this->response->setJSON([
+            'ok' => empty($result['errors'] ?? []),
+            'result' => $result,
+            'sql_count' => $sqlCount,
+            'applied' => $applied,
+            'apply_error_count' => $applyErrors,
+            'should_continue' => $shouldContinue,
+            'message' => $shouldContinue
+                ? 'Applied chunk, continuing...'
+                : 'Schema sync auto-apply reached a stop condition.',
+        ]);
+    }
+
+    /**
+     * @return array<string, mixed>
+     */
+    private function executeSchemaSyncAction(string $action): array
+    {
         try {
-            $action = strtolower(trim((string) $this->request->getPost('sync_action')));
+            $maxRuntimeSeconds = (int) env('setup.sync.max_runtime_seconds', 40);
-            if (!in_array($action, ['analyze', 'apply'], true)) {
+            if ($maxRuntimeSeconds < 5) {
-                $action = 'analyze';
+                $maxRuntimeSeconds = 5;
             }
 
-            $syncResult = $this->buildSchemaSyncPlan();
+            $startedAt = microtime(true);
+
+            $syncResult = $this->buildSchemaSyncPlan($startedAt, $maxRuntimeSeconds);
             if ($action === 'apply' && empty($syncResult['errors']) && !empty($syncResult['sql'])) {
-                $applyResult = $this->applySchemaSyncSql($syncResult['sql']);
+                $maxApplyStatements = (int) env('setup.sync.max_apply_statements_per_run', 75);
+                if ($maxApplyStatements <= 0) {
+                    $maxApplyStatements = 75;
+                }
+
+                $applyResult = $this->applySchemaSyncSql($syncResult['sql'], $startedAt, $maxRuntimeSeconds, $maxApplyStatements);
                 $syncResult['applied'] = $applyResult['applied'];
                 $syncResult['apply_errors'] = $applyResult['errors'];
+                $syncResult['apply_truncated'] = $applyResult['truncated'];
             }
 
-            $this->lastSyncResult = $syncResult;
+            return $syncResult;
         } catch (\Throwable $e) {
             log_message('error', 'Setup schemaSync failed: {message}', ['message' => $e->getMessage()]);
-            $this->lastSyncResult = [
+            return [
                 'errors' => ['Schema sync failed: ' . $e->getMessage()],
                 'sql' => [],
                 'summary' => [],
                 'source' => 'runtime-exception',
             ];
         }
-
-        return view('setup/db_setup', $this->buildDbToolsViewData());
     }
 
     public function exportMasterSchema()
@@ -344,6 +400,33 @@ class Setup extends BaseController
         return redirect()->to($this->dbToolsUrlWithKey());
     }
 
+    public function prepareFilesystem()
+    {
+        if ($deny = $this->ensureSetupAccess()) {
+            return $deny;
+        }
+
+        $result = $this->prepareWritablePaths();
+
+        $parts = [];
+        $parts[] = 'Filesystem prep done.';
+        $parts[] = 'Created: ' . count($result['created']) . '.';
+        $parts[] = 'Permission-updated: ' . count($result['permission_updated']) . '.';
+        $parts[] = 'Writable: ' . count($result['writable']) . '/' . count($result['checked']) . '.';
+
+        if (!empty($result['failed'])) {
+            $parts[] = 'Failed: ' . implode(', ', $result['failed']) . '.';
+        }
+
+        if (!empty($result['commands'])) {
+            $parts[] = 'Run on server shell if needed: ' . implode(' | ', $result['commands']);
+        }
+
+        session()->setFlashdata('setup_msg', implode(' ', $parts));
+
+        return redirect()->to($this->dbToolsUrlWithKey());
+    }
+
     /**
      * @return array<string, mixed>
      */
@@ -425,13 +508,23 @@ class Setup extends BaseController
             $errors[] = 'Database error: ' . $dbError;
         }
 
-        $logPath = WRITEPATH . 'logs';
+        foreach ($this->requiredWritablePaths() as $path) {
-        if (!is_dir($logPath) || !is_writable($logPath)) {
+            $relative = $this->relativeProjectPath($path);
-            $warnings[] = 'writable/logs is not writable. Error diagnostics may be incomplete.';
+            if (!is_dir($path)) {
-        } else {
+                $warnings[] = $relative . ' does not exist.';
-            $info[] = 'Logs path: ' . $logPath;
+                continue;
+            }
+
+            if (!is_writable($path)) {
+                $warnings[] = $relative . ' is not writable.';
+            } else {
+                $info[] = $relative . ' is writable.';
+            }
         }
 
+        $info[] = 'Web SAPI/PHP: ' . PHP_SAPI . ' / ' . PHP_VERSION;
+        $info[] = 'If spark fails but web works, your CLI PHP version likely differs from web PHP version.';
+
         if ($this->isSetupLocked()) {
             $warnings[] = 'Setup lock file exists. Remove lock file if this is a fresh installation attempt.';
         }
@@ -446,7 +539,7 @@ class Setup extends BaseController
     /**
      * @return array<string, mixed>
      */
-    private function buildSchemaSyncPlan(): array
+    private function buildSchemaSyncPlan(float $startedAt, int $maxRuntimeSeconds): array
     {
         $errors = [];
         $sql = [];
@@ -491,7 +584,13 @@ class Setup extends BaseController
                 break;
             }
 
+            if ((microtime(true) - $startedAt) >= $maxRuntimeSeconds) {
+                $truncated = true;
+                break;
+            }
+
             $table = (string) $table;
+            $processedTables++;
             if (!isset($clientTables[$table])) {
                 $createSql = (string) ($masterTableDef['create_sql'] ?? '');
                 if ($createSql !== '') {
@@ -515,7 +614,6 @@ class Setup extends BaseController
                 $alterCount += count($tableAlterSql);
             }
 
-            $processedTables++;
         }
 
         return [
@@ -535,19 +633,32 @@ class Setup extends BaseController
 
     /**
      * @param array<int, string> $sqlList
-     * @return array{applied:int,errors:array<int,string>}
+     * @return array{applied:int,errors:array<int,string>,truncated:bool}
      */
-    private function applySchemaSyncSql(array $sqlList): array
+    private function applySchemaSyncSql(array $sqlList, float $startedAt, int $maxRuntimeSeconds, int $maxStatements): array
     {
         $client = $this->resolveClientConnection();
         $applied = 0;
         $errors = [];
+        $truncated = false;
+        $attempted = 0;
 
         foreach ($sqlList as $stmt) {
+            if ($attempted >= $maxStatements) {
+                $truncated = true;
+                break;
+            }
+
+            if ((microtime(true) - $startedAt) >= $maxRuntimeSeconds) {
+                $truncated = true;
+                break;
+            }
+
             $stmt = $this->normalizeSyncSql((string) $stmt);
             if ($stmt === '') {
                 continue;
             }
+            $attempted++;
             try {
                 $client->query($stmt);
                 $applied++;
@@ -575,7 +686,7 @@ class Setup extends BaseController
             }
         }
 
-        return ['applied' => $applied, 'errors' => $errors];
+        return ['applied' => $applied, 'errors' => $errors, 'truncated' => $truncated];
     }
 
     /**
@@ -1054,4 +1165,86 @@ class Setup extends BaseController
         $value = ucwords(strtolower(trim($value)));
         return str_replace(' ', '', $value);
     }
+
+    /**
+     * @return array<int, string>
+     */
+    private function requiredWritablePaths(): array
+    {
+        return [
+            WRITEPATH,
+            WRITEPATH . 'cache',
+            WRITEPATH . 'logs',
+            WRITEPATH . 'session',
+            WRITEPATH . 'uploads',
+            WRITEPATH . 'debugbar',
+            WRITEPATH . 'tmp',
+            FCPATH . 'uploads',
+        ];
+    }
+
+    /**
+     * @return array<string, array<int, string>>
+     */
+    private function prepareWritablePaths(): array
+    {
+        $checked = [];
+        $created = [];
+        $permissionUpdated = [];
+        $writable = [];
+        $failed = [];
+
+        $paths = $this->requiredWritablePaths();
+        foreach ($paths as $path) {
+            $checked[] = $this->relativeProjectPath($path);
+
+            if (!is_dir($path)) {
+                if (@mkdir($path, 0775, true)) {
+                    $created[] = $this->relativeProjectPath($path);
+                }
+            }
+
+            if (is_dir($path) && !is_writable($path)) {
+                if (@chmod($path, 0775)) {
+                    $permissionUpdated[] = $this->relativeProjectPath($path);
+                }
+            }
+
+            if (is_dir($path) && is_writable($path)) {
+                $writable[] = $this->relativeProjectPath($path);
+            } else {
+                $failed[] = $this->relativeProjectPath($path);
+            }
+        }
+
+        $commands = [];
+        if (!empty($failed)) {
+            $projectPath = rtrim(str_replace('\\', '/', ROOTPATH), '/');
+            $commands[] = 'sudo chown -R www-data:www-data "' . $projectPath . '/writable" "' . $projectPath . '/public/uploads"';
+            $commands[] = 'sudo find "' . $projectPath . '/writable" -type d -exec chmod 775 {} \;';
+            $commands[] = 'sudo find "' . $projectPath . '/writable" -type f -exec chmod 664 {} \;';
+            $commands[] = 'sudo chmod -R 775 "' . $projectPath . '/public/uploads"';
+        }
+
+        return [
+            'checked' => $checked,
+            'created' => $created,
+            'permission_updated' => $permissionUpdated,
+            'writable' => $writable,
+            'failed' => array_values(array_unique($failed)),
+            'commands' => $commands,
+        ];
+    }
+
+    private function relativeProjectPath(string $absolutePath): string
+    {
+        $normalizedPath = str_replace('\\', '/', $absolutePath);
+        $normalizedRoot = str_replace('\\', '/', ROOTPATH);
+
+        if (str_starts_with($normalizedPath, $normalizedRoot)) {
+            return ltrim(substr($normalizedPath, strlen($normalizedRoot)), '/');
+        }
+
+        return $absolutePath;
+    }
 }

app/Views/setup/db_setup.php

@@ -86,9 +86,22 @@
                 <div class="form-group">
                     <button type="submit" name="sync_action" value="analyze" class="btn btn-info">Analyze Schema Diff</button>
                     <button type="submit" name="sync_action" value="apply" class="btn btn-warning" onclick="return confirm('Apply generated SQL to client DB?');">Apply Sync to Client DB</button>
+                    <button type="button" id="auto-sync-btn" class="btn btn-primary">Auto Apply Until Done</button>
+                    <button type="button" id="auto-sync-stop-btn" class="btn btn-default" disabled>Stop Auto Apply</button>
                 </div>
             </form>
 
+            <div id="auto-sync-status" class="alert alert-info" style="display:none;"></div>
+
+            <form method="post" action="<?= base_url('setup/db-tools/prepare-filesystem') ?>" style="margin-top:10px; margin-bottom:10px;">
+                <?= csrf_field() ?>
+                <?php if (!empty($setup_key ?? '')) : ?>
+                    <input type="hidden" name="key" value="<?= esc($setup_key) ?>">
+                <?php endif; ?>
+                <button type="submit" class="btn btn-default" onclick="return confirm('Create/fix required writable folders now?');">Prepare Writable Folders</button>
+                <span class="text-muted" style="margin-left:10px;">Ensures writable/cache, logs, session, uploads, debugbar, tmp, public/uploads.</span>
+            </form>
+
             <?php if (!empty($sync_result ?? null)) : ?>
                 <?php if (!empty($sync_result['errors'] ?? [])) : ?>
                     <div class="alert alert-danger">
@@ -124,6 +137,14 @@
                     </div>
                 <?php endif; ?>
 
+                <?php if (!empty($sync_result['apply_truncated'] ?? false)) : ?>
+                    <div class="alert alert-warning">
+                        Apply phase was truncated to prevent request timeout.
+                        Re-run <strong>Apply Sync to Client DB</strong> to continue remaining statements,
+                        or increase <code>setup.sync.max_apply_statements_per_run</code> / <code>setup.sync.max_runtime_seconds</code> in .env.
+                    </div>
+                <?php endif; ?>
+
                 <div class="form-group">
                     <label>Generated SQL</label>
                     <textarea class="form-control" rows="14" readonly><?php
@@ -160,3 +181,140 @@
         </div>
     </div>
 </section>
+
+<script>
+    (function () {
+        var autoBtn = document.getElementById('auto-sync-btn');
+        var stopBtn = document.getElementById('auto-sync-stop-btn');
+        var statusBox = document.getElementById('auto-sync-status');
+        if (!autoBtn || !stopBtn || !statusBox) {
+            return;
+        }
+
+        var running = false;
+        var cancelRequested = false;
+        var maxRounds = 30;
+
+        function setStatus(html, klass) {
+            statusBox.style.display = 'block';
+            statusBox.className = 'alert ' + (klass || 'alert-info');
+            statusBox.innerHTML = html;
+        }
+
+        function readCsrf() {
+            var tokenInput = document.querySelector('input[name="<?= esc(csrf_token()) ?>"]');
+            return tokenInput ? tokenInput.value : '';
+        }
+
+        function buildFormBody() {
+            var csrfName = '<?= esc(csrf_token()) ?>';
+            var csrfValue = readCsrf();
+            var parts = [];
+            if (csrfValue) {
+                parts.push(encodeURIComponent(csrfName) + '=' + encodeURIComponent(csrfValue));
+            }
+            <?php if (!empty($setup_key ?? '')) : ?>
+            parts.push('key=' + encodeURIComponent('<?= esc($setup_key) ?>'));
+            <?php endif; ?>
+            return parts.join('&');
+        }
+
+        async function runRound(round) {
+            var res = await fetch('<?= base_url('setup/db-tools/schema-sync-step') ?>', {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
+                    'X-Requested-With': 'XMLHttpRequest'
+                },
+                body: buildFormBody(),
+                credentials: 'same-origin'
+            });
+
+            if (!res.ok) {
+                throw new Error('HTTP ' + res.status);
+            }
+
+            return await res.json();
+        }
+
+        autoBtn.addEventListener('click', async function () {
+            if (running) {
+                return;
+            }
+
+            if (!confirm('Auto-apply schema sync in multiple safe chunks?')) {
+                return;
+            }
+
+            running = true;
+            cancelRequested = false;
+            autoBtn.disabled = true;
+            stopBtn.disabled = false;
+            setStatus('Starting auto apply...', 'alert-info');
+
+            try {
+                var rounds = 0;
+                var totalApplied = 0;
+
+                while (rounds < maxRounds) {
+                    if (cancelRequested) {
+                        setStatus('Auto apply stopped by user after ' + rounds + ' rounds. You can resume anytime.', 'alert-warning');
+                        break;
+                    }
+
+                    rounds++;
+                    setStatus('Running round ' + rounds + ' of ' + maxRounds + '...', 'alert-info');
+
+                    var data = await runRound(rounds);
+                    var result = data.result || {};
+                    var appliedNow = parseInt(data.applied || 0, 10);
+                    var sqlCount = parseInt(data.sql_count || 0, 10);
+                    var applyErrCount = parseInt(data.apply_error_count || 0, 10);
+                    totalApplied += appliedNow;
+
+                    if (!data.ok) {
+                        var errList = (result.errors || []).join('<br>');
+                        setStatus('Stopped: ' + (errList || 'Schema sync step failed.'), 'alert-danger');
+                        break;
+                    }
+
+                    if (applyErrCount > 0) {
+                        var applyErrors = (result.apply_errors || []).slice(0, 5).join('<br>');
+                        setStatus('Stopped due to apply errors after ' + rounds + ' rounds.<br>' + applyErrors, 'alert-danger');
+                        break;
+                    }
+
+                    if (!data.should_continue) {
+                        var msg = 'Completed after ' + rounds + ' rounds. Total applied: ' + totalApplied + '. Last round SQL: ' + sqlCount + '.';
+                        setStatus(msg + ' Reloading page...', 'alert-success');
+                        setTimeout(function () {
+                            window.location.reload();
+                        }, 900);
+                        break;
+                    }
+
+                    setStatus('Round ' + rounds + ' done. Applied this round: ' + appliedNow + '. Total applied: ' + totalApplied + '. Continuing...', 'alert-info');
+                }
+
+                if (rounds >= maxRounds) {
+                    setStatus('Stopped after max rounds (' + maxRounds + '). Click again to continue from current state.', 'alert-warning');
+                }
+            } catch (error) {
+                setStatus('Auto apply failed: ' + (error && error.message ? error.message : 'Unknown error'), 'alert-danger');
+            } finally {
+                running = false;
+                autoBtn.disabled = false;
+                stopBtn.disabled = true;
+            }
+        });
+
+        stopBtn.addEventListener('click', function () {
+            if (!running) {
+                return;
+            }
+            cancelRequested = true;
+            stopBtn.disabled = true;
+            setStatus('Stop requested. Finishing current request...', 'alert-warning');
+        });
+    })();
+</script>