fix: fallback to /raw if contentlength not available from head

extensions/weavedrive/src/index.js

@@ -71,14 +71,27 @@ module.exports = function weaveDrive(mod, FS) {
       const node = FS.createFile('/', 'data/' + id, properties, true, false)
 
       // Set initial parameters
-      const response = await this.customFetch(`/${id}`, {
+      let dataPath = `/${id}`
+      let response = await this.customFetch(dataPath, {
         method: 'HEAD',
         headers: { 'Accept-Encoding': 'identity' }
       })
-      if (!response.ok) {
+      let bytesLength = response.ok ? response.headers.get('Content-Length') : null
+
+      if (!response.ok || bytesLength === null || Number.isNaN(Number(bytesLength))) {
+        dataPath = `/raw/${id}`
+        response = await this.customFetch(dataPath, {
+          method: 'HEAD',
+          headers: { 'Accept-Encoding': 'identity' }
+        })
+        bytesLength = response.ok ? response.headers.get('Content-Length') : null
+      }
+
+      if (!response.ok || bytesLength === null || Number.isNaN(Number(bytesLength))) {
         return 'HALT'
       }
-      const bytesLength = response.headers.get('Content-Length')
+
+      node.gatewayPath = dataPath
       node.total_size = Number(bytesLength)
       node.cache = new Uint8Array(0)
       node.position = 0
@@ -234,7 +247,7 @@ module.exports = function weaveDrive(mod, FS) {
       // console.log("WeaveDrive: fd: ", fd, " Read length: ", toRead, " Reading ahead:", to - toRead - stream.position)
 
       // Fetch with streaming
-      const response = await this.customFetch(`/${stream.node.name}`, {
+      const response = await this.customFetch(stream.node.gatewayPath || `/${stream.node.name}`, {
         method: 'GET',
         redirect: 'follow',
         headers: { Range: `bytes=${stream.position}-${to}` }

extensions/weavedrive/test/index.test.js

@@ -477,6 +477,84 @@ test('boot loader set to tx id', async function () {
   assert.equal(result.Output.data, '<TICKER>')
 })
 
+describe('data path selection', () => {
+  function response({ ok = true, contentLength = null, chunks = [] } = {}) {
+    return {
+      ok,
+      headers: {
+        get(name) {
+          return name.toLowerCase() === 'content-length' ? contentLength : null
+        }
+      },
+      body: {
+        getReader() {
+          let index = 0
+          return {
+            async read() {
+              if (index >= chunks.length) return { done: true }
+              return { done: false, value: chunks[index++] }
+            },
+            releaseLock() {}
+          }
+        }
+      }
+    }
+  }
+
+  function fakeFs() {
+    const streams = []
+    let node
+
+    return {
+      streams,
+      analyzePath() {
+        return { exists: false }
+      },
+      mkdir() {},
+      createFile(_root, path, properties) {
+        node = { ...properties, name: path.split('/').pop() }
+        return node
+      },
+      open(path) {
+        const stream = { fd: streams.length, path, node, position: node.position || 0 }
+        streams[stream.fd] = stream
+        return stream
+      }
+    }
+  }
+
+  test('falls back to raw path when id path has no length', async () => {
+    const fetches = []
+    const FS = fakeFs()
+    const mod = { HEAP8: new Uint8Array(8) }
+    const wd = weaveDrive(mod, FS)
+
+    wd.checkAdmissible = async () => true
+    wd.customFetch = async (path, options) => {
+      fetches.push({ path, options })
+      if (options.method === 'HEAD' && path === '/TX') return response()
+      if (options.method === 'HEAD' && path === '/raw/TX') return response({ contentLength: '4' })
+      if (options.method === 'GET' && path === '/raw/TX') {
+        return response({ chunks: [new Uint8Array([1, 2, 3, 4])] })
+      }
+      throw new Error(`unexpected fetch: ${options.method} ${path}`)
+    }
+
+    const stream = await wd.create('TX')
+    assert.equal(stream.node.gatewayPath, '/raw/TX')
+    assert.equal(stream.node.total_size, 4)
+
+    const bytesRead = await wd.read(stream.fd, 0, 4)
+
+    assert.equal(bytesRead, 4)
+    assert.deepEqual(Array.from(mod.HEAP8.slice(0, 4)), [1, 2, 3, 4])
+    assert.deepEqual(
+      fetches.map(({ path, options }) => `${options.method} ${path}`),
+      ['HEAD /TX', 'HEAD /raw/TX', 'GET /raw/TX']
+    )
+  })
+})
+
 describe('joinUrl', () => {
   const wd = weaveDrive()
   const joinUrl = wd.joinUrl.bind(wd)