fetch.bac

'
' Small utility to get some BaCon programs over HTTP from the BaCon website
'
' Dec 2010 - PvE.
' Mar 2011: added size check - PvE
' Jul 2013: various improvements, network handling more robust - PvE
' January 2017: option collapse to skip empty results - PvE
' Dec 2018: support for HTTP proxy, fix error in coloring, code optimizations - PvE
' July 2019: skip comments in files.txt
' Sept 2019: use plain HTTP only
'-----------------------------------------------------------------------------------

OPTION COLLAPSE TRUE

CONST Separator$ = CR$ & NL$ & CR$ & NL$

CONST website$ = "www.basic-converter.org"

CONST NetDelay = 500

'-----------------------------------------------------------------------------------------

FUNCTION Connect_To(name$)

    LOCAL net TYPE int

    ' Get environment variable for proxy
    proxy$ = GETENVIRON$("HTTP_PROXY")
    IF LEN(proxy$) = 0 THEN proxy$ = GETENVIRON$("http_proxy")

    ' Parse proxy settings
    IF LEN(proxy$) THEN
        IF LEFT$(proxy$, 7) <> "http://" THEN
            EPRINT "System error: proxy '", proxy$, "' not supported! Redefine to default HTTP proxy or undefine."
            END 1
        ENDIF

        proxy$ = MID$(proxy$, 8)
        IF TALLY(proxy$, "@") THEN
            unpw$ = B64ENC$(TOKEN$(proxy$, 1, "@"))
            proxy$ = TOKEN$(proxy$, 2, "@")
        ENDIF

        ' Setup TCP connection on defined port
        OPEN proxy$ FOR NETWORK AS net

        SEND "CONNECT " & name$ & ":80 HTTP/1.1" & "\r\n" TO net
        IF LEN(unpw$) THEN SEND "Proxy-Authorization: Basic " & B64ENC$(unpw$) & "\r\n" TO net
        SEND "\r\n" TO net

        ' Return should be OK
        RECEIVE dat$ FROM net
        IF NOT(TALLY(dat$, " 200 ")) THEN
            EPRINT "System error: proxy '", proxy$, "' returns error: " & dat$
            END 1
        ENDIF
    ELSE
        ' Open TCP connection on port 80
        OPEN name$ & ":80" FOR NETWORK AS net
    ENDIF

    RETURN net

ENDFUNCTION

'-----------------------------------------------------------------------------------------

SUB Close_Fd(int net)

    CLOSE NETWORK net

END SUB

'-----------------------------------------------------------------------------------------

' Get the list with files
PRINT "Fetching the list of BaCon programs... ";

mynet = Connect_To(website$)

SEND "GET /files.txt HTTP/1.1\r\nHost: " & website$ & Separator$ TO mynet

REPEAT
    RECEIVE dat$ FROM mynet
    IF LEN(dat$) > 0 THEN allfile$ = allfile$ & dat$
UNTIL ISFALSE(WAIT(mynet, NetDelay))

Close_Fd(mynet)

COLOR FG TO GREEN
PRINT "done."

'-----------------------------------------------------------------------------------------

' Split the text into separate filenames and fetch one by one

FOR file$ IN TOKEN$(allfile$, 2, Separator$) STEP NL$

    ' Skip comments
    IF LEFT$(file$, 1) = "#" THEN CONTINUE

    ' Give the remote server some time
    SLEEP NetDelay

    COLOR RESET
    PRINT "Fetching ", file$, "... ";
    attempt = 0

    REPEAT
        ' We create a new connection for each individual file
        mynet = Connect_To(website$)

        ' Get the current file
        SEND "GET /" & file$ & " HTTP/1.1\r\nHost: " & website$ & Separator$ TO mynet
        total$ = ""

        ' Prevent endless waiting for server
        IF NOT(WAIT(mynet, NetDelay)) THEN
            COLOR FG TO CYAN
            PRINT "Timeout on server! Retrying... ";
            SLEEP NetDelay
            Close_Fd(mynet)
            INCR attempt
            IF attempt = 4 THEN
                COLOR FG TO RED
                PRINT "Could not download ", file$, " - skipping."
                SLEEP NetDelay
                BREAK
            END IF
            CONTINUE
        ELSE
            ' So there is data
            RECEIVE dat$ FROM mynet

            ' Get the filesize from the HTTP header
            dat$ = MID$(dat$, INSTR(dat$, "Content-Length:")+15)
            length = VAL(LEFT$(dat$, INSTR(dat$, NL$)))

            ' As long as there is data, get it
            WHILE WAIT(mynet, NetDelay) AND LEN(dat$) > 0
                total$ = total$ & dat$
                RECEIVE dat$ FROM mynet
            WEND

            total$ = total$ & dat$

            ' Write to file
            SAVE TOKEN$(total$, 2, Separator$) TO file$

            ' Closing the connection again
            Close_Fd(mynet)
        END IF

        ' If the size of the file is not correct
        IF FILELEN(file$) <> length THEN

            ' Keep track of attempts
            INCR attempt

            SELECT attempt
                CASE 2, 3
                    COLOR FG TO CYAN
                    PRINT "Failed! Retrying... ";
                    SLEEP NetDelay
                CASE 4
                    COLOR FG TO RED
                    PRINT "Could not download ", file$, " - skipping."
                    SLEEP NetDelay
                    IF FILEEXISTS(file$) THEN DELETE FILE file$
                    BREAK
            END SELECT
        END IF

    ' Make sure filelength equals size mentioned in HTTP header
    UNTIL LEN(total$) > 0 AND FILELEN(file$) = length

    IF attempt < 4 THEN
        COLOR FG TO GREEN
        PRINT "done."
    END IF

NEXT

COLOR RESET

PRINT "Finished list of BaCon demonstration programs. Enjoy!"