fix: download DuckDB extensions over HTTPS with retries

scripts/embed_duckdb_ext/main.go

@@ -8,6 +8,7 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"time"
 
 	_ "github.com/duckdb/duckdb-go/v2"
 )
@@ -21,6 +22,11 @@ var platforms = []string{"linux_amd64", "linux_arm64", "osx_amd64", "osx_arm64"}
 // Embed directory to store the extensions
 var embedDirRoot = "runtime/drivers/duckdb/extensions/embed/"
 
+// Extensions are tens of MB each; the timeout bounds a stalled connection, not a slow one.
+var httpClient = &http.Client{Timeout: 10 * time.Minute}
+
+const downloadAttempts = 4
+
 // main downloads and embeds the necessary DuckDB extensions for the current DuckDB version and pre-defined platforms
 // https://duckdb.org/docs/extensions/working_with_extensions.html
 func main() {
@@ -54,7 +60,7 @@ func main() {
 
 		for _, extension := range extensions {
 			// Make sure an embedded extension is compressed (gzipped) to save space
-			url := fmt.Sprintf("http://extensions.duckdb.org/%s/%s/%s.duckdb_extension.gz", duckdbVersion, platform, extension)
+			url := fmt.Sprintf("https://extensions.duckdb.org/%s/%s/%s.duckdb_extension.gz", duckdbVersion, platform, extension)
 			destPath := filepath.Join(destDir, fmt.Sprintf("%s.duckdb_extension.gz", extension))
 
 			// Check if the extension already exists and skip downloading
@@ -76,9 +82,26 @@ func main() {
 	log.Println("All necessary DuckDB extensions have been processed.")
 }
 
-// downloadFile downloads a file from a URL and saves it to a destination path
+// downloadFile downloads a file from a URL and saves it to a destination path, retrying transient failures
 func downloadFile(url, dest string) error {
-	resp, err := http.Get(url)
+	var err error
+	for attempt := 1; attempt <= downloadAttempts; attempt++ {
+		err = downloadOnce(url, dest)
+		if err == nil {
+			return nil
+		}
+		if attempt < downloadAttempts {
+			log.Printf("Attempt %d/%d to download %s failed: %v", attempt, downloadAttempts, url, err)
+			time.Sleep(time.Duration(attempt) * 5 * time.Second)
+		}
+	}
+	return err
+}
+
+// downloadOnce writes to a temporary file and renames it into place, so an interrupted
+// download never leaves a partial file that a later run would skip as already present.
+func downloadOnce(url, dest string) error {
+	resp, err := httpClient.Get(url)
 	if err != nil {
 		return fmt.Errorf("failed to perform HTTP GET request: %w", err)
 	}
@@ -88,16 +111,25 @@ func downloadFile(url, dest string) error {
 		return fmt.Errorf("unexpected HTTP status: %s", resp.Status)
 	}
 
-	out, err := os.Create(dest)
+	tmp := dest + ".tmp"
+	out, err := os.Create(tmp)
 	if err != nil {
 		return fmt.Errorf("failed to create file: %w", err)
 	}
-	defer out.Close()
 
 	_, err = io.Copy(out, resp.Body)
+	if closeErr := out.Close(); err == nil {
+		err = closeErr
+	}
 	if err != nil {
+		os.Remove(tmp)
 		return fmt.Errorf("failed to copy response body to file: %w", err)
 	}
 
+	if err := os.Rename(tmp, dest); err != nil {
+		os.Remove(tmp)
+		return fmt.Errorf("failed to move downloaded file into place: %w", err)
+	}
+
 	return nil
 }