Refactor DAV client, add missing operations

.github/workflows/dav-integration.yml

@@ -0,0 +1,146 @@
+name: DAV Integration Tests
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+    - ".github/workflows/dav-integration.yml"
+    - "dav/**"
+    - "go.mod"
+    - "go.sum"
+  push:
+    branches:
+      - main
+
+concurrency:
+  group: dav-integration
+  cancel-in-progress: false
+
+jobs:
+  dav-integration:
+    name: DAV Integration Tests
+    runs-on: ubuntu-latest
+
+    services:
+      webdav:
+        image: httpd:2.4
+        ports:
+          - 8443:443
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: go.mod
+
+      - name: Install Ginkgo
+        run: go install github.com/onsi/ginkgo/v2/ginkgo@latest
+
+      - name: Setup WebDAV Server Configuration
+        run: |
+          # Create certificates
+          mkdir -p /tmp/webdav-certs
+          openssl req -x509 -nodes -days 365 -newkey rsa:2048 \
+            -keyout /tmp/webdav-certs/server.key \
+            -out /tmp/webdav-certs/server.crt \
+            -subj "/C=US/ST=Test/L=Test/O=Test/CN=localhost" \
+            -addext "subjectAltName=DNS:localhost,IP:127.0.0.1"
+
+          # Create WebDAV directory
+          mkdir -p /tmp/webdav-data
+          chmod 777 /tmp/webdav-data
+
+          # Create htpasswd file
+          docker run --rm httpd:2.4 htpasswd -nb testuser testpass > /tmp/webdav.passwd
+
+          # Create Apache config with DAV
+          cat > /tmp/httpd.conf << 'EOF'
+          ServerRoot "/usr/local/apache2"
+          Listen 443
+
+          LoadModule mpm_event_module modules/mod_mpm_event.so
+          LoadModule authn_file_module modules/mod_authn_file.so
+          LoadModule authn_core_module modules/mod_authn_core.so
+          LoadModule authz_host_module modules/mod_authz_host.so
+          LoadModule authz_user_module modules/mod_authz_user.so
+          LoadModule authz_core_module modules/mod_authz_core.so
+          LoadModule auth_basic_module modules/mod_auth_basic.so
+          LoadModule dav_module modules/mod_dav.so
+          LoadModule dav_fs_module modules/mod_dav_fs.so
+          LoadModule setenvif_module modules/mod_setenvif.so
+          LoadModule ssl_module modules/mod_ssl.so
+          LoadModule unixd_module modules/mod_unixd.so
+          LoadModule dir_module modules/mod_dir.so
+
+          User daemon
+          Group daemon
+
+          DAVLockDB /usr/local/apache2/var/DavLock
+
+          <IfModule ssl_module>
+              SSLRandomSeed startup builtin
+              SSLRandomSeed connect builtin
+          </IfModule>
+
+          <VirtualHost *:443>
+              SSLEngine on
+              SSLCertificateFile /usr/local/apache2/certs/server.crt
+              SSLCertificateKeyFile /usr/local/apache2/certs/server.key
+
+              DocumentRoot "/usr/local/apache2/webdav"
+
+              <Directory "/usr/local/apache2/webdav">
+                  Dav On
+                  Options +Indexes
+                  AuthType Basic
+                  AuthName "WebDAV"
+                  AuthUserFile /usr/local/apache2/webdav.passwd
+                  Require valid-user
+
+                  <LimitExcept GET OPTIONS>
+                      Require valid-user
+                  </LimitExcept>
+              </Directory>
+          </VirtualHost>
+          EOF
+
+          # Get the service container ID
+          CONTAINER_ID=$(docker ps --filter "ancestor=httpd:2.4" --format "{{.ID}}")
+          echo "WebDAV container ID: $CONTAINER_ID"
+
+          # Create required directories in container first
+          docker exec $CONTAINER_ID mkdir -p /usr/local/apache2/certs
+          docker exec $CONTAINER_ID mkdir -p /usr/local/apache2/webdav
+          docker exec $CONTAINER_ID mkdir -p /usr/local/apache2/var
+          docker exec $CONTAINER_ID chmod 777 /usr/local/apache2/webdav
+          docker exec $CONTAINER_ID chmod 777 /usr/local/apache2/var
+
+          # Copy files to container
+          docker cp /tmp/httpd.conf $CONTAINER_ID:/usr/local/apache2/conf/httpd.conf
+          docker cp /tmp/webdav.passwd $CONTAINER_ID:/usr/local/apache2/webdav.passwd
+          docker cp /tmp/webdav-certs/server.crt $CONTAINER_ID:/usr/local/apache2/certs/server.crt
+          docker cp /tmp/webdav-certs/server.key $CONTAINER_ID:/usr/local/apache2/certs/server.key
+
+          # Reload Apache
+          docker exec $CONTAINER_ID apachectl graceful
+
+          # Wait for Apache to be ready
+          sleep 5
+
+          # Test connection
+          curl -k -u testuser:testpass -v https://localhost:8443/ || echo "WebDAV server not ready yet"
+
+      - name: Run Integration Tests
+        env:
+          DAV_ENDPOINT: "https://localhost:8443"
+          DAV_USER: "testuser"
+          DAV_PASSWORD: "testpass"
+          DAV_SECRET: "test-secret-key"
+        run: |
+          export DAV_CA_CERT="$(cat /tmp/webdav-certs/server.crt)"
+          cd dav
+          ginkgo -v ./integration
+

.github/workflows/unit-test.yml

@@ -42,7 +42,7 @@ jobs:
               run: |
                     export CGO_ENABLED=0
                     go version
-                    go test -v ./dav/...
+                    go run github.com/onsi/ginkgo/v2/ginkgo --skip-package=integration ./dav/...
 
             - name: gcs unit tests
               run: |

dav/README.md

@@ -8,24 +8,67 @@ For general usage and build instructions, see the [main README](../README.md).
 
 ## DAV-Specific Configuration
 
-The DAV client requires a JSON configuration file with WebDAV endpoint details and credentials.
+The DAV client requires a JSON configuration file with the following structure:
+
+``` json
+{
+  "endpoint":        "<string> (required)",
+  "user":            "<string> (optional)",
+  "password":        "<string> (optional)",
+  "retry_attempts":  <uint> (optional - default: 3),
+  "tls": {
+    "cert": {
+      "ca":          "<string> (optional - PEM-encoded CA certificate)"
+    }
+  },
+  "secret":          "<string> (optional - required for pre-signed URLs)"
+}
+```
 
 **Usage examples:**
 ```bash
-# Upload an object
-storage-cli -s dav -c dav-config.json put local-file.txt remote-object
+# Upload a blob
+storage-cli -s dav -c dav-config.json put local-file.txt remote-blob
+
+# Fetch a blob (destination file will be overwritten if exists)
+storage-cli -s dav -c dav-config.json get remote-blob local-file.txt
+
+# Delete a blob
+storage-cli -s dav -c dav-config.json delete remote-blob
+
+# Check if blob exists
+storage-cli -s dav -c dav-config.json exists remote-blob
+
+# List all blobs
+storage-cli -s dav -c dav-config.json list
 
-# Fetch an object
-storage-cli -s dav -c dav-config.json get remote-object local-file.txt
+# List blobs with prefix
+storage-cli -s dav -c dav-config.json list my-prefix
 
-# Delete an object
-storage-cli -s dav -c dav-config.json delete remote-object
+# Copy a blob
+storage-cli -s dav -c dav-config.json copy source-blob destination-blob
 
-# Check if an object exists
-storage-cli -s dav -c dav-config.json exists remote-object
+# Delete blobs by prefix
+storage-cli -s dav -c dav-config.json delete-recursive my-prefix-
 
-# Generate a signed URL (e.g., GET for 1 hour)
-storage-cli -s dav -c dav-config.json sign remote-object get 60s
+# Get blob properties (outputs JSON with ContentLength, ETag, LastModified)
+storage-cli -s dav -c dav-config.json properties remote-blob
+
+# Ensure storage exists (initialize WebDAV storage)
+storage-cli -s dav -c dav-config.json ensure-storage-exists
+
+# Generate a pre-signed URL (e.g., GET for 3600 seconds)
+storage-cli -s dav -c dav-config.json sign remote-blob get 3600s
+```
+
+### Using Signed URLs with curl
+
+```bash
+# Downloading a blob:
+curl -X GET <signed-url>
+
+# Uploading a blob:
+curl -X PUT -T path/to/file <signed-url>
 ```
 
 ## Pre-signed URLs
@@ -38,12 +81,76 @@ The HMAC format is:
 `<HTTP Verb><Object ID><Unix timestamp of the signature time><Unix timestamp of the expiration time>`
 
 The generated URL format:
-`https://blobstore.url/signed/object-id?st=HMACSignatureHash&ts=GenerationTimestamp&e=ExpirationTimestamp`
+`https://blobstore.url/signed/8c/object-id?st=HMACSignatureHash&ts=GenerationTimestamp&e=ExpirationTime`
+
+**Note:** The `/8c/` represents the SHA1 prefix directory where the blob is stored. Pre-signed URLs require the WebDAV server to have signature verification middleware. Standard WebDAV servers don't support this - it's a Cloud Foundry extension.
+
+## Features
+
+### SHA1-Based Prefix Directories
+All blobs are stored in subdirectories based on the first 2 hex characters of their SHA1 hash (e.g., blob `my-file.txt` → path `/8c/my-file.txt`). This distributes files across 256 directories (00-ff) to prevent performance issues with large flat directories.
+
+### Automatic Retry Logic
+All operations automatically retry on transient errors with 1-second delays between attempts. Default is 3 retry attempts, configurable via `retry_attempts` in config.
+
+### TLS/HTTPS Support
+Supports HTTPS connections with custom CA certificates for internal or self-signed certificates.
 
 ## Testing
 
 ### Unit Tests
 Run unit tests from the repository root:
+
 ```bash
-ginkgo --cover -v -r ./dav/...
+ginkgo --cover -v -r ./dav/client
 ```
+
+Or using go test:
+```bash
+go test ./dav/client/...
+```
+
+### Integration Tests
+
+The DAV implementation includes Go-based integration tests that run against a real WebDAV server.
+
+**Prerequisites:**
+- Running WebDAV server (can be set up with Docker - see below)
+- Environment variables configured
+
+**Setup WebDAV server with Docker:**
+```bash
+cd dav
+./setup-webdav-test.sh   # Sets up Apache WebDAV with HTTPS
+```
+
+**Run integration tests:**
+```bash
+# Set environment variables (from the dav/ directory after running setup)
+export DAV_ENDPOINT="https://localhost:8443"
+export DAV_USER="testuser"
+export DAV_PASSWORD="testpass"
+export DAV_CA_CERT="$(cat webdav-test/certs/ca.pem)"
+export DAV_SECRET="test-secret-key"  # Optional, for signed URL tests
+
+# Run integration tests
+ginkgo -v ./integration
+
+# Or using go test
+go test -v ./integration/...
+```
+
+These tests cover all operations: PUT, GET, DELETE, DELETE-RECURSIVE, EXISTS, LIST, COPY, PROPERTIES, and ENSURE-STORAGE-EXISTS.
+
+### End-to-End Tests
+
+The DAV implementation also includes shell-based end-to-end tests using the compiled storage-cli binary.
+
+**Quick start:**
+```bash
+cd dav
+./setup-webdav-test.sh   # Sets up Apache WebDAV with HTTPS
+./test-storage-cli.sh     # Runs complete test suite
+```
+
+**For detailed testing instructions, see [TESTING.md](TESTING.md).**