Add GitLab export diagnostics command

README.md

@@ -119,6 +119,41 @@ The `--gitlab-server-url` flag accepts both GitLab.com (`https://gitlab.com`) an
 
 5. The `migrate.ps1` script requires PowerShell to run. If not already installed see the [install instructions](https://docs.microsoft.com/en-us/powershell/scripting/install/installing-powershell?view=powershell-7.2) to install PowerShell on Windows, Linux, or Mac. Then run the script.
 
+### GitLab export diagnostics
+
+GitLab-to-GitHub migration does not require an administrator SSH key. If an export fails before a GitHub migration ID is created, collect API-visible diagnostics with your `GITLAB_PAT`:
+
+```bash
+gh gl2gh diagnose-gitlab-export \
+  --gitlab-server-url https://gitlab.example.com \
+  --gitlab-group parent/group --gitlab-project project \
+  --output diagnostics.md
+```
+
+Use the same GitLab user that initiated the export: the export API status is user-specific. No GitHub PAT or migration ID is required for this command.
+
+For self-managed GitLab, optionally add administrator SSH access to collect actual export job/child job IDs, retained errors and matching server log entries:
+
+```bash
+gh gl2gh diagnose-gitlab-export \
+  --gitlab-server-url https://gitlab.example.com \
+  --gitlab-group parent/group --gitlab-project project \
+  --output diagnostics.md \
+  --ssh-host gitlab-admin.example.com --ssh-user admin \
+  --ssh-key /path/to/private-key --ssh-port 22
+```
+
+If GitLab runs in Docker on that SSH host, add `--gitlab-container gitlab`. Omit it if SSH already lands inside the GitLab container. SSH must reach the **OS administrator shell**, not GitLab's Git-over-SSH endpoint.
+
+- Install the OpenSSH client (`ssh`) on the machine running the CLI. Verify the server's host key through a trusted channel and add it to your OpenSSH `known_hosts` before running the command. Unknown or changed keys are rejected; host verification is never disabled.
+- Supply `--ssh-host`, `--ssh-user` and `--ssh-key` together. Encrypted keys must already be unlocked in `ssh-agent`; SSH password/passphrase prompts are disabled.
+- The account must be root or have non-interactive `sudo` access to `gitlab-rails` (or `docker exec` for container installations). Rails runner executes an administrator script and Docker access is effectively root access; use an appropriately authorized account.
+- Collection is read-only: it does not start/retry exports or change GitLab settings. It collects the latest 10 export jobs across users, up to 100 relations per job, and the last 8 MiB/100 matching entries of each current `exporter.log`, Sidekiq, `exceptions_json.log` and `api_json.log`. Strings and backtraces are abbreviated. Missing files, unavailable version-specific records and truncation appear as warnings in the report.
+- Collection targets Linux-package GitLab installations, directly or inside Docker, with a five-minute SSH limit. Rotated logs, other worker nodes and centralized/Kubernetes logging are not collected automatically; use the administrator follow-up instructions when the report is incomplete.
+- If SSH collection fails, the command exits with an error and preserves the API-only report. Use `--overwrite` to replace an existing report.
+
+**Treat reports and verbose CLI logs as sensitive.** Server messages can include customer data, internal paths and credentials. Collection limits which log fields are retained, but does not guarantee secret redaction. Reports are written with owner-only permissions on Linux/macOS; on Windows, secure the output directory with appropriate ACLs. Store all files securely and review/redact them before sharing. The private SSH key stays on the client; only its file path is passed to OpenSSH.
+
 ### Skipping version checks
 
 When the CLI is launched, it logs if a newer version of the CLI is available. You can skip this check by setting the `GEI_SKIP_VERSION_CHECK` environment variable to `true`. 

RELEASENOTES.md

@@ -1 +1 @@
-
+- GitLab: Added `diagnose-gitlab-export` to collect export status and project statistics, with optional administrator SSH access to retrieve server-side export errors and logs from Linux-package or Docker installations.

src/Octoshift/Services/FileSystemProvider.cs

@@ -21,6 +21,22 @@ public class FileSystemProvider
 
     public virtual async Task WriteAllTextAsync(string path, string contents) => await File.WriteAllTextAsync(path, contents);
 
+    public virtual async Task WritePrivateTextAsync(string path, string contents)
+    {
+        var options = new FileStreamOptions { Mode = FileMode.Create, Access = FileAccess.Write };
+        if (!OperatingSystem.IsWindows())
+        {
+            options.UnixCreateMode = UnixFileMode.UserRead | UnixFileMode.UserWrite;
+        }
+        await using var stream = new FileStream(path, options);
+        if (!OperatingSystem.IsWindows())
+        {
+            File.SetUnixFileMode(stream.SafeFileHandle, UnixFileMode.UserRead | UnixFileMode.UserWrite);
+        }
+        await using var writer = new StreamWriter(stream);
+        await writer.WriteAsync(contents);
+    }
+
     public virtual async ValueTask WriteAsync(FileStream fileStream, ReadOnlyMemory<byte> buffer, CancellationToken cancellationToken = default)
     {
         if (fileStream is null)

src/Octoshift/Services/GitlabApi.cs

@@ -65,6 +65,41 @@ public virtual async Task<string> StartExport(string groupPath, string projectPa
         );
     }
 
+    public virtual async Task<GitlabExportDetails> GetExportDetails(string groupPath, string projectPath)
+    {
+        var encodedProjectPath = GetEncodedProjectPath(groupPath, projectPath);
+        var url = $"{_gitlabBaseUrl}/api/v4/projects/{encodedProjectPath}/export";
+
+        var exportResponse = await _client.GetAsync(url);
+        var exportData = JObject.Parse(exportResponse);
+
+        return new GitlabExportDetails(
+            (long?)exportData["id"],
+            (string)exportData["export_status"],
+            (string)exportData["_links"]?["api_url"],
+            exportData.ToString());
+    }
+
+    public virtual async Task<GitlabProjectDetails> GetProjectDetails(string groupPath, string projectPath)
+    {
+        var encodedProjectPath = GetEncodedProjectPath(groupPath, projectPath);
+        var url = $"{_gitlabBaseUrl}/api/v4/projects/{encodedProjectPath}?statistics=true";
+
+        var projectResponse = await _client.GetAsync(url);
+        var projectData = JObject.Parse(projectResponse);
+        var projectStatistics = (JObject)projectData["statistics"];
+
+        return new GitlabProjectDetails(
+            (long?)projectData["id"],
+            (string)projectData["path_with_namespace"],
+            (string)projectData["web_url"],
+            (bool?)projectData["archived"],
+            (string)projectData["visibility"],
+            (long?)projectStatistics?["repository_size"],
+            (long?)projectStatistics?["uploads_size"],
+            (long?)projectStatistics?["job_artifacts_size"]);
+    }
+
     public virtual async Task DownloadExportArchive(string groupPath, string projectPath, string file)
     {
         var encodedProjectPath = GetEncodedProjectPath(groupPath, projectPath);
@@ -168,3 +203,15 @@ private static string GetEncodedProjectPath(string groupPath, string projectPath
         return pathWithNamespace.EscapeDataString();
     }
 }
+
+public record GitlabExportDetails(long? Id, string ExportStatus, string DownloadUrl, string RawJson);
+
+public record GitlabProjectDetails(
+    long? Id,
+    string PathWithNamespace,
+    string WebUrl,
+    bool? Archived,
+    string Visibility,
+    long? RepositorySize,
+    long? UploadsSize,
+    long? JobArtifactsSize);

src/OctoshiftCLI.Tests/gl2gh/Commands/DiagnoseGitlabExport/DiagnoseGitlabExportCommandArgsTests.cs

@@ -0,0 +1,28 @@
+using FluentAssertions;
+using OctoshiftCLI.GitlabToGithub.Commands.DiagnoseGitlabExport;
+using OctoshiftCLI.Services;
+using Xunit;
+
+namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.DiagnoseGitlabExport;
+
+public class DiagnoseGitlabExportCommandArgsTests
+{
+    private readonly OctoLogger _log = new();
+
+    [Theory]
+    [InlineData(null, "group", "project", "--gitlab-server-url must be provided.")]
+    [InlineData("https://gitlab.contoso.com", null, "project", "--gitlab-group must be provided.")]
+    [InlineData("https://gitlab.contoso.com", "group", null, "--gitlab-project must be provided.")]
+    public void Validate_Requires_Gitlab_Project_Inputs(string gitlabServerUrl, string gitlabGroup, string gitlabProject, string expectedMessage)
+    {
+        var args = new DiagnoseGitlabExportCommandArgs
+        {
+            GitlabServerUrl = gitlabServerUrl,
+            GitlabGroup = gitlabGroup,
+            GitlabProject = gitlabProject
+        };
+
+        var ex = Assert.Throws<OctoshiftCliException>(() => args.Validate(_log));
+        ex.Message.Should().Be(expectedMessage);
+    }
+}

src/OctoshiftCLI.Tests/gl2gh/Commands/DiagnoseGitlabExport/DiagnoseGitlabExportCommandHandlerTests.cs

@@ -0,0 +1,125 @@
+using System.Threading.Tasks;
+using Moq;
+using OctoshiftCLI.GitlabToGithub.Commands.DiagnoseGitlabExport;
+using OctoshiftCLI.GitlabToGithub.Services;
+using OctoshiftCLI.Services;
+using Xunit;
+
+namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.DiagnoseGitlabExport;
+
+public class DiagnoseGitlabExportCommandHandlerTests
+{
+    private readonly Mock<OctoLogger> _mockOctoLogger = TestHelpers.CreateMock<OctoLogger>();
+    private readonly Mock<GitlabApi> _mockGitlabApi = TestHelpers.CreateMock<GitlabApi>();
+    private readonly Mock<FileSystemProvider> _mockFileSystemProvider = TestHelpers.CreateMock<FileSystemProvider>();
+    private readonly Mock<GitlabSshDiagnosticsCollector> _mockSshCollector = new();
+
+    private readonly DiagnoseGitlabExportCommandHandler _handler;
+
+    public DiagnoseGitlabExportCommandHandlerTests()
+    {
+        _handler = new DiagnoseGitlabExportCommandHandler(_mockOctoLogger.Object, _mockGitlabApi.Object, _mockFileSystemProvider.Object, _mockSshCollector.Object);
+    }
+
+    [Fact]
+    public async Task Handle_Writes_Report_With_Export_Status_And_Gitlab_Admin_Commands()
+    {
+        var args = new DiagnoseGitlabExportCommandArgs
+        {
+            GitlabServerUrl = "https://gitlab.contoso.com",
+            GitlabGroup = "parent/group",
+            GitlabProject = "project",
+            Output = "diagnostics.md"
+        };
+        string report = null;
+
+        _mockGitlabApi.Setup(m => m.GetServerVersion()).ReturnsAsync(("18.11.0-ee", true));
+        _mockGitlabApi.Setup(m => m.GetProjectDetails("parent/group", "project"))
+            .ReturnsAsync(new GitlabProjectDetails(123, "parent/group/project", "https://gitlab.contoso.com/parent/group/project", false, "private", 42, 43, 44));
+        _mockGitlabApi.Setup(m => m.GetExportDetails("parent/group", "project"))
+            .ReturnsAsync(new GitlabExportDetails(123, "failed", null, "{\"export_status\":\"failed\"}"));
+        _mockFileSystemProvider.Setup(m => m.WritePrivateTextAsync("diagnostics.md", It.IsAny<string>()))
+            .Callback<string, string>((_, contents) => report = contents)
+            .Returns(Task.CompletedTask);
+
+        await _handler.Handle(args);
+
+        Assert.Contains("Export status: failed", report);
+        Assert.Contains("p.export_jobs", report);
+        Assert.DoesNotContain("p.import_state", report);
+        Assert.DoesNotContain("Export ID:", report);
+        Assert.Contains("same user that initiated", report);
+        Assert.Contains("/var/log/gitlab/sidekiq/current", report);
+        Assert.Contains("/var/log/gitlab/gitlab-rails/exporter.log", report);
+        _mockSshCollector.Verify(m => m.Collect(It.IsAny<DiagnoseGitlabExportCommandArgs>()), Times.Never);
+        _mockOctoLogger.Verify(m => m.LogWarning(It.Is<string>(s => s.Contains("GitLab reported the project export as failed"))), Times.Once);
+        _mockOctoLogger.Verify(m => m.LogSuccess("Wrote GitLab export diagnostics to diagnostics.md."), Times.Once);
+    }
+
+    [Theory]
+    [InlineData(false, false)]
+    [InlineData(true, false)]
+    [InlineData(false, true)]
+    public async Task Handle_Collects_Ssh_Only_When_Requested_And_Preserves_Api_Report_On_Failure(bool fail, bool wrongProject)
+    {
+        var args = new DiagnoseGitlabExportCommandArgs
+        {
+            GitlabServerUrl = "http://gitlab",
+            GitlabGroup = "group",
+            GitlabProject = "project",
+            Output = "diagnostics.md",
+            SshHost = "admin-host"
+        };
+        _mockGitlabApi.Setup(m => m.GetServerVersion()).ReturnsAsync(("18.3.1", false));
+        _mockGitlabApi.Setup(m => m.GetProjectDetails("group", "project"))
+            .ReturnsAsync(new GitlabProjectDetails(1, "group/project", "http://gitlab/group/project", false, "private", 1, 0, 0));
+        _mockGitlabApi.Setup(m => m.GetExportDetails("group", "project"))
+            .ReturnsAsync(new GitlabExportDetails(1, "failed", null, "{\"export_status\":\"failed\"}"));
+        string report = null;
+        _mockFileSystemProvider.Setup(m => m.WritePrivateTextAsync(args.Output, It.IsAny<string>()))
+            .Callback<string, string>((_, contents) => report = contents).Returns(Task.CompletedTask);
+        if (fail)
+        {
+            _mockSshCollector.Setup(m => m.Collect(args)).ThrowsAsync(new OctoshiftCliException("SSH failed"));
+        }
+        else
+        {
+            _mockSshCollector.Setup(m => m.Collect(args))
+                .ReturnsAsync($"{{\"project_id\":{(wrongProject ? 2 : 1)},\"warnings\":[\"log missing\"],\"error\":\"Permission denied ```\"}}");
+        }
+        if (fail || wrongProject)
+        {
+            await Assert.ThrowsAsync<OctoshiftCliException>(() => _handler.Handle(args));
+            Assert.Contains("Collection failed", report);
+            _mockOctoLogger.Verify(m => m.LogSuccess(It.IsAny<string>()), Times.Never);
+        }
+        else
+        {
+            await _handler.Handle(args);
+            Assert.Contains("Permission denied \\u0060\\u0060\\u0060", report);
+            _mockOctoLogger.Verify(m => m.LogWarning(It.Is<string>(s => s.Contains("collection warnings"))), Times.Once);
+        }
+        Assert.Contains("Export status: failed", report);
+        Assert.Contains("## Server-side diagnostics (SSH)", report);
+        _mockSshCollector.Verify(m => m.Collect(args), Times.Once);
+        _mockFileSystemProvider.Verify(m => m.WritePrivateTextAsync(args.Output, It.IsAny<string>()), Times.Exactly(2));
+    }
+
+    [Fact]
+    public async Task Handle_Throws_When_Output_Exists_Without_Overwrite()
+    {
+        var args = new DiagnoseGitlabExportCommandArgs
+        {
+            GitlabServerUrl = "https://gitlab.contoso.com",
+            GitlabGroup = "parent/group",
+            GitlabProject = "project",
+            Output = "diagnostics.md"
+        };
+
+        _mockFileSystemProvider.Setup(m => m.FileExists("diagnostics.md")).Returns(true);
+
+        var ex = await Assert.ThrowsAsync<OctoshiftCliException>(() => _handler.Handle(args));
+
+        Assert.Equal("File diagnostics.md already exists! Use --overwrite to overwrite this file.", ex.Message);
+    }
+}

src/OctoshiftCLI.Tests/gl2gh/Commands/DiagnoseGitlabExport/DiagnoseGitlabExportCommandTests.cs

@@ -0,0 +1,68 @@
+using System;
+using FluentAssertions;
+using Moq;
+using OctoshiftCLI.GitlabToGithub.Commands.DiagnoseGitlabExport;
+using OctoshiftCLI.GitlabToGithub.Factories;
+using OctoshiftCLI.GitlabToGithub.Services;
+using OctoshiftCLI.Services;
+using Xunit;
+
+namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.DiagnoseGitlabExport;
+
+public class DiagnoseGitlabExportCommandTests
+{
+    private const string GITLAB_SERVER_URL = "https://gitlab.contoso.com";
+    private const string GITLAB_PAT = "gitlab-pat";
+
+    private readonly Mock<IServiceProvider> _mockServiceProvider = new();
+    private readonly Mock<GitlabApiFactory> _mockGitlabApiFactory = TestHelpers.CreateMock<GitlabApiFactory>();
+    private readonly Mock<OctoLogger> _mockOctoLogger = TestHelpers.CreateMock<OctoLogger>();
+    private readonly Mock<FileSystemProvider> _mockFileSystemProvider = TestHelpers.CreateMock<FileSystemProvider>();
+
+    private readonly DiagnoseGitlabExportCommand _command = [];
+
+    public DiagnoseGitlabExportCommandTests()
+    {
+        _mockServiceProvider.Setup(m => m.GetService(typeof(OctoLogger))).Returns(_mockOctoLogger.Object);
+        _mockServiceProvider.Setup(m => m.GetService(typeof(GitlabApiFactory))).Returns(_mockGitlabApiFactory.Object);
+        _mockServiceProvider.Setup(m => m.GetService(typeof(FileSystemProvider))).Returns(_mockFileSystemProvider.Object);
+        _mockServiceProvider.Setup(m => m.GetService(typeof(GitlabSshDiagnosticsCollector))).Returns(new GitlabSshDiagnosticsCollector());
+    }
+
+    [Fact]
+    public void Should_Have_Options()
+    {
+        _command.Should().NotBeNull();
+        _command.Name.Should().Be("diagnose-gitlab-export");
+        _command.Options.Count.Should().Be(13);
+
+        TestHelpers.VerifyCommandOption(_command.Options, "gitlab-server-url", false);
+        TestHelpers.VerifyCommandOption(_command.Options, "gitlab-group", false);
+        TestHelpers.VerifyCommandOption(_command.Options, "gitlab-project", false);
+        TestHelpers.VerifyCommandOption(_command.Options, "gitlab-pat", false);
+        TestHelpers.VerifyCommandOption(_command.Options, "output", false);
+        TestHelpers.VerifyCommandOption(_command.Options, "overwrite", false);
+        TestHelpers.VerifyCommandOption(_command.Options, "no-ssl-verify", false);
+        TestHelpers.VerifyCommandOption(_command.Options, "verbose", false);
+        TestHelpers.VerifyCommandOption(_command.Options, "ssh-host", false);
+        TestHelpers.VerifyCommandOption(_command.Options, "ssh-user", false);
+        TestHelpers.VerifyCommandOption(_command.Options, "ssh-key", false);
+        TestHelpers.VerifyCommandOption(_command.Options, "ssh-port", false);
+        TestHelpers.VerifyCommandOption(_command.Options, "gitlab-container", false);
+    }
+
+    [Fact]
+    public void It_Creates_The_GitlabApi_With_The_Provided_Server_Url_And_Pat()
+    {
+        var args = new DiagnoseGitlabExportCommandArgs
+        {
+            GitlabServerUrl = GITLAB_SERVER_URL,
+            GitlabPat = GITLAB_PAT,
+            NoSslVerify = true
+        };
+
+        _command.BuildHandler(args, _mockServiceProvider.Object);
+
+        _mockGitlabApiFactory.Verify(m => m.Create(GITLAB_SERVER_URL, GITLAB_PAT, true));
+    }
+}