diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 1f9d6b8c..7c955c41 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -1,4 +1,4 @@ -name: Close and mark stale issue +name: Close Stale Issues on: schedule: diff --git a/benchmarks/import/package.json b/benchmarks/import/package.json index 2c380ca6..af01dd55 100644 --- a/benchmarks/import/package.json +++ b/benchmarks/import/package.json @@ -13,7 +13,7 @@ }, "devDependencies": { "aegir": "^47.0.16", - "blockstore-core": "^5.0.4", + "blockstore-core": "^6.0.2", "ipfs-unixfs-importer": "^15.0.0", "it-buffer-stream": "^3.0.11", "it-drain": "^3.0.10" diff --git a/benchmarks/memory/package.json b/benchmarks/memory/package.json index 70f46164..3b4c0055 100644 --- a/benchmarks/memory/package.json +++ b/benchmarks/memory/package.json @@ -13,7 +13,7 @@ }, "devDependencies": { "aegir": "^47.0.16", - "blockstore-fs": "^2.0.4", + "blockstore-fs": "^3.0.1", "ipfs-unixfs-importer": "^15.0.0", "it-drain": "^3.0.10" }, diff --git a/packages/ipfs-unixfs-exporter/CHANGELOG.md b/packages/ipfs-unixfs-exporter/CHANGELOG.md index f11662da..463832da 100644 --- a/packages/ipfs-unixfs-exporter/CHANGELOG.md +++ b/packages/ipfs-unixfs-exporter/CHANGELOG.md @@ -1,3 +1,33 @@ +## [ipfs-unixfs-exporter-v13.7.3](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.7.2...ipfs-unixfs-exporter-13.7.3) (2025-08-12) + +### Bug Fixes + +* export basic file from dir or shard ([#440](https://github.com/ipfs/js-ipfs-unixfs/issues/440)) ([b8d33de](https://github.com/ipfs/js-ipfs-unixfs/commit/b8d33deb0dfc76cc53eb82e31a67748a8da24eae)) + +## [ipfs-unixfs-exporter-v13.7.2](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.7.1...ipfs-unixfs-exporter-13.7.2) (2025-07-31) + +### Bug Fixes + +* add extended to default exporter options ([#439](https://github.com/ipfs/js-ipfs-unixfs/issues/439)) ([278aea4](https://github.com/ipfs/js-ipfs-unixfs/commit/278aea4c2ed76a8d890a0d2d3a079b03a9c00334)) + +## [ipfs-unixfs-exporter-v13.7.1](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.7.0...ipfs-unixfs-exporter-13.7.1) (2025-07-31) + +### Bug Fixes + +* add option to export non-extended unixfs ([#438](https://github.com/ipfs/js-ipfs-unixfs/issues/438)) ([c9a9bf4](https://github.com/ipfs/js-ipfs-unixfs/commit/c9a9bf45a5c8a779ed73cc2238a58c01e090edb7)) + +## [ipfs-unixfs-exporter-v13.7.0](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.6.6...ipfs-unixfs-exporter-13.7.0) (2025-07-30) + +### Features + +* add 'extended' option to exporter ([#437](https://github.com/ipfs/js-ipfs-unixfs/issues/437)) ([332a794](https://github.com/ipfs/js-ipfs-unixfs/commit/332a794227f7792e1ddee1b1e47d01fd510d6cf4)) + +## [ipfs-unixfs-exporter-v13.6.6](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.6.5...ipfs-unixfs-exporter-13.6.6) (2025-06-18) + +### Bug Fixes + +* constrain the unixfs type ([#435](https://github.com/ipfs/js-ipfs-unixfs/issues/435)) ([7663b87](https://github.com/ipfs/js-ipfs-unixfs/commit/7663b87ed2e3e8cd4da1484ca601638740ea0ae7)) + ## [ipfs-unixfs-exporter-v13.6.5](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.6.4...ipfs-unixfs-exporter-13.6.5) (2025-06-18) ### Documentation diff --git a/packages/ipfs-unixfs-exporter/package.json b/packages/ipfs-unixfs-exporter/package.json index 2110db9e..2d6219dc 100644 --- a/packages/ipfs-unixfs-exporter/package.json +++ b/packages/ipfs-unixfs-exporter/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-unixfs-exporter", - "version": "13.6.5", + "version": "13.7.3", "description": "JavaScript implementation of the UnixFs exporter used by IPFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/main/packages/ipfs-unixfs-exporter#readme", @@ -143,7 +143,7 @@ "@ipld/dag-pb": "^4.1.5", "@multiformats/murmur3": "^2.1.8", "hamt-sharding": "^3.0.6", - "interface-blockstore": "^5.3.2", + "interface-blockstore": "^6.0.1", "ipfs-unixfs": "^11.0.0", "it-filter": "^3.1.4", "it-last": "^3.0.9", @@ -151,15 +151,16 @@ "it-parallel": "^3.0.13", "it-pipe": "^3.0.1", "it-pushable": "^3.2.3", + "it-to-buffer": "^4.0.10", "multiformats": "^13.3.7", - "p-queue": "^8.1.0", + "p-queue": "^9.0.0", "progress-events": "^1.0.1" }, "devDependencies": { "@types/readable-stream": "^4.0.21", "@types/sinon": "^17.0.4", "aegir": "^47.0.16", - "blockstore-core": "^5.0.4", + "blockstore-core": "^6.0.2", "delay": "^6.0.0", "ipfs-unixfs-importer": "^15.0.0", "iso-random-stream": "^2.0.2", @@ -167,7 +168,6 @@ "it-buffer-stream": "^3.0.11", "it-drain": "^3.0.10", "it-first": "^3.0.9", - "it-to-buffer": "^4.0.10", "merge-options": "^3.0.4", "readable-stream": "^4.7.0", "sinon": "^21.0.0", diff --git a/packages/ipfs-unixfs-exporter/src/index.ts b/packages/ipfs-unixfs-exporter/src/index.ts index e8b06f59..35e92277 100644 --- a/packages/ipfs-unixfs-exporter/src/index.ts +++ b/packages/ipfs-unixfs-exporter/src/index.ts @@ -134,6 +134,34 @@ export interface ExporterOptions extends ProgressOptions * (default: undefined) */ blockReadConcurrency?: number + + /** + * When directory contents are listed, by default the root node of each entry + * is fetched to decode the UnixFS metadata and know if the entry is a file or + * a directory. This can result in fetching extra data which may not be + * desirable, depending on your application. + * + * Pass false here to only return the CID and the name of the entry and not + * any extended metadata. + * + * @default true + */ + extended?: boolean +} + +export interface BasicExporterOptions extends ExporterOptions { + /** + * When directory contents are listed, by default the root node of each entry + * is fetched to decode the UnixFS metadata and know if the entry is a file or + * a directory. This can result in fetching extra data which may not be + * desirable, depending on your application. + * + * Pass false here to only return the CID and the name of the entry and not + * any extended metadata. + * + * @default true + */ + extended: false } export interface Exportable { @@ -218,7 +246,7 @@ export interface Exportable { * // `entries` contains the first 5 files/directories in the directory * ``` */ - content(options?: ExporterOptions): AsyncGenerator + content(options?: ExporterOptions | BasicExporterOptions): AsyncGenerator } /** @@ -298,6 +326,23 @@ export interface IdentityNode extends Exportable { */ export type UnixFSEntry = UnixFSFile | UnixFSDirectory | ObjectNode | RawNode | IdentityNode +export interface UnixFSBasicEntry { + /** + * The name of the entry + */ + name: string + + /** + * The path of the entry within the DAG in which it was encountered + */ + path: string + + /** + * The CID of the entry + */ + cid: CID +} + export interface NextResult { cid: CID name: string @@ -311,12 +356,20 @@ export interface ResolveResult { } export interface Resolve { (cid: CID, name: string, path: string, toResolve: string[], depth: number, blockstore: ReadableStorage, options: ExporterOptions): Promise } -export interface Resolver { (cid: CID, name: string, path: string, toResolve: string[], resolve: Resolve, depth: number, blockstore: ReadableStorage, options: ExporterOptions): Promise } +export interface Resolver { (cid: CID, name: string, path: string, toResolve: string[], resolve: Resolve, depth: number, blockstore: ReadableStorage, options: ExporterOptions | BasicExporterOptions): Promise } export type UnixfsV1FileContent = AsyncIterable | Iterable export type UnixfsV1DirectoryContent = AsyncIterable | Iterable export type UnixfsV1Content = UnixfsV1FileContent | UnixfsV1DirectoryContent -export interface UnixfsV1Resolver { (cid: CID, node: PBNode, unixfs: UnixFS, path: string, resolve: Resolve, depth: number, blockstore: ReadableStorage): (options: ExporterOptions) => UnixfsV1Content } + +export interface UnixFsV1ContentResolver { + (options: ExporterOptions): UnixfsV1Content + (options: BasicExporterOptions): UnixFSBasicEntry +} + +export interface UnixfsV1Resolver { + (cid: CID, node: PBNode, unixfs: UnixFS, path: string, resolve: Resolve, depth: number, blockstore: ReadableStorage): (options: ExporterOptions) => UnixfsV1Content +} export interface ShardTraversalContext { hamtDepth: number @@ -387,6 +440,8 @@ const cidAndRest = (path: string | Uint8Array | CID): { cid: CID, toResolve: str * // entries contains 4x `entry` objects * ``` */ +export function walkPath (path: string | CID, blockstore: ReadableStorage, options?: ExporterOptions): AsyncGenerator +export function walkPath (path: string | CID, blockstore: ReadableStorage, options: BasicExporterOptions): AsyncGenerator export async function * walkPath (path: string | CID, blockstore: ReadableStorage, options: ExporterOptions = {}): AsyncGenerator { let { cid, @@ -443,6 +498,8 @@ export async function * walkPath (path: string | CID, blockstore: ReadableStorag * } * ``` */ +export async function exporter (path: string | CID, blockstore: ReadableStorage, options?: ExporterOptions): Promise +export async function exporter (path: string | CID, blockstore: ReadableStorage, options: BasicExporterOptions): Promise export async function exporter (path: string | CID, blockstore: ReadableStorage, options: ExporterOptions = {}): Promise { const result = await last(walkPath(path, blockstore, options)) @@ -471,6 +528,8 @@ export async function exporter (path: string | CID, blockstore: ReadableStorage, * // entries contains all children of the `Qmfoo/foo/bar` directory and it's children * ``` */ +export function recursive (path: string | CID, blockstore: ReadableStorage, options?: ExporterOptions): AsyncGenerator +export function recursive (path: string | CID, blockstore: ReadableStorage, options: BasicExporterOptions): AsyncGenerator export async function * recursive (path: string | CID, blockstore: ReadableStorage, options: ExporterOptions = {}): AsyncGenerator { const node = await exporter(path, blockstore, options) diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/dag-cbor.ts b/packages/ipfs-unixfs-exporter/src/resolvers/dag-cbor.ts index 0631af98..323f4255 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/dag-cbor.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/dag-cbor.ts @@ -1,9 +1,10 @@ import * as dagCbor from '@ipld/dag-cbor' +import toBuffer from 'it-to-buffer' import { resolveObjectPath } from '../utils/resolve-object-path.js' import type { Resolver } from '../index.js' const resolve: Resolver = async (cid, name, path, toResolve, resolve, depth, blockstore, options) => { - const block = await blockstore.get(cid, options) + const block = await toBuffer(blockstore.get(cid, options)) const object = dagCbor.decode(block) return resolveObjectPath(object, block, cid, name, path, toResolve, depth) diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/dag-json.ts b/packages/ipfs-unixfs-exporter/src/resolvers/dag-json.ts index c206d7af..a8ed7dae 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/dag-json.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/dag-json.ts @@ -1,9 +1,10 @@ import * as dagJson from '@ipld/dag-json' +import toBuffer from 'it-to-buffer' import { resolveObjectPath } from '../utils/resolve-object-path.js' import type { Resolver } from '../index.js' const resolve: Resolver = async (cid, name, path, toResolve, resolve, depth, blockstore, options) => { - const block = await blockstore.get(cid, options) + const block = await toBuffer(blockstore.get(cid, options)) const object = dagJson.decode(block) return resolveObjectPath(object, block, cid, name, path, toResolve, depth) diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/json.ts b/packages/ipfs-unixfs-exporter/src/resolvers/json.ts index 90c5a174..92c2c852 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/json.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/json.ts @@ -1,9 +1,10 @@ +import toBuffer from 'it-to-buffer' import * as json from 'multiformats/codecs/json' import { resolveObjectPath } from '../utils/resolve-object-path.js' import type { Resolver } from '../index.js' const resolve: Resolver = async (cid, name, path, toResolve, resolve, depth, blockstore, options) => { - const block = await blockstore.get(cid, options) + const block = await toBuffer(blockstore.get(cid, options)) const object = json.decode(block) return resolveObjectPath(object, block, cid, name, path, toResolve, depth) diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/raw.ts b/packages/ipfs-unixfs-exporter/src/resolvers/raw.ts index 4082e388..509d46ac 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/raw.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/raw.ts @@ -1,3 +1,4 @@ +import toBuffer from 'it-to-buffer' import { CustomProgressEvent } from 'progress-events' import { NotFoundError } from '../errors.js' import extractDataFromBlock from '../utils/extract-data-from-block.js' @@ -30,7 +31,7 @@ const resolve: Resolver = async (cid, name, path, toResolve, resolve, depth, blo throw new NotFoundError(`No link named ${path} found in raw node ${cid}`) } - const block = await blockstore.get(cid, options) + const block = await toBuffer(blockstore.get(cid, options)) return { entry: { diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/directory.ts b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/directory.ts index afab2634..73dc5c06 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/directory.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/directory.ts @@ -3,10 +3,11 @@ import map from 'it-map' import parallel from 'it-parallel' import { pipe } from 'it-pipe' import { CustomProgressEvent } from 'progress-events' -import type { ExporterOptions, ExportWalk, UnixfsV1DirectoryContent, UnixfsV1Resolver } from '../../../index.js' +import { isBasicExporterOptions } from '../../../utils/is-basic-exporter-options.ts' +import type { BasicExporterOptions, ExporterOptions, ExportWalk, UnixFSBasicEntry, UnixfsV1Resolver } from '../../../index.js' const directoryContent: UnixfsV1Resolver = (cid, node, unixfs, path, resolve, depth, blockstore) => { - async function * yieldDirectoryContent (options: ExporterOptions = {}): UnixfsV1DirectoryContent { + async function * yieldDirectoryContent (options: ExporterOptions | BasicExporterOptions = {}): any { const offset = options.offset ?? 0 const length = options.length ?? node.Links.length const links = node.Links.slice(offset, length) @@ -21,6 +22,17 @@ const directoryContent: UnixfsV1Resolver = (cid, node, unixfs, path, resolve, de return async () => { const linkName = link.Name ?? '' const linkPath = `${path}/${linkName}` + + if (isBasicExporterOptions(options)) { + const basic: UnixFSBasicEntry = { + cid: link.Hash, + name: linkName, + path: linkPath + } + + return basic + } + const result = await resolve(link.Hash, linkName, linkPath, [], depth + 1, blockstore, options) return result.entry } diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/file.ts b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/file.ts index 3d549048..7cac1738 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/file.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/file.ts @@ -4,6 +4,7 @@ import map from 'it-map' import parallel from 'it-parallel' import { pipe } from 'it-pipe' import { pushable } from 'it-pushable' +import toBuffer from 'it-to-buffer' import * as raw from 'multiformats/codecs/raw' import PQueue from 'p-queue' import { CustomProgressEvent } from 'progress-events' @@ -76,7 +77,7 @@ async function walkDAG (blockstore: ReadableStorage, node: dagPb.PBNode | Uint8A childOps, (source) => map(source, (op) => { return async () => { - const block = await blockstore.get(op.link.Hash, options) + const block = await toBuffer(blockstore.get(op.link.Hash, options)) return { ...op, diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts index b08255a2..1fafb07c 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts @@ -3,13 +3,15 @@ import { UnixFS } from 'ipfs-unixfs' import map from 'it-map' import parallel from 'it-parallel' import { pipe } from 'it-pipe' +import toBuffer from 'it-to-buffer' import { CustomProgressEvent } from 'progress-events' import { NotUnixFSError } from '../../../errors.js' -import type { ExporterOptions, Resolve, UnixfsV1DirectoryContent, UnixfsV1Resolver, ReadableStorage, ExportWalk } from '../../../index.js' +import { isBasicExporterOptions } from '../../../utils/is-basic-exporter-options.ts' +import type { ExporterOptions, Resolve, UnixfsV1DirectoryContent, UnixfsV1Resolver, ReadableStorage, ExportWalk, BasicExporterOptions, UnixFSBasicEntry } from '../../../index.js' import type { PBNode } from '@ipld/dag-pb' const hamtShardedDirectoryContent: UnixfsV1Resolver = (cid, node, unixfs, path, resolve, depth, blockstore) => { - function yieldHamtDirectoryContent (options: ExporterOptions = {}): UnixfsV1DirectoryContent { + function yieldHamtDirectoryContent (options: ExporterOptions | BasicExporterOptions = {}): UnixfsV1DirectoryContent { options.onProgress?.(new CustomProgressEvent('unixfs:exporter:walk:hamt-sharded-directory', { cid })) @@ -20,7 +22,7 @@ const hamtShardedDirectoryContent: UnixfsV1Resolver = (cid, node, unixfs, path, return yieldHamtDirectoryContent } -async function * listDirectory (node: PBNode, path: string, resolve: Resolve, depth: number, blockstore: ReadableStorage, options: ExporterOptions): UnixfsV1DirectoryContent { +async function * listDirectory (node: PBNode, path: string, resolve: Resolve, depth: number, blockstore: ReadableStorage, options: ExporterOptions | BasicExporterOptions): any { const links = node.Links if (node.Data == null) { @@ -47,19 +49,41 @@ async function * listDirectory (node: PBNode, path: string, resolve: Resolve, de const name = link.Name != null ? link.Name.substring(padLength) : null if (name != null && name !== '') { - const result = await resolve(link.Hash, name, `${path}/${name}`, [], depth + 1, blockstore, options) + const linkPath = `${path}/${name}` - return { entries: result.entry == null ? [] : [result.entry] } + if (isBasicExporterOptions(options)) { + const basic: UnixFSBasicEntry = { + cid: link.Hash, + name, + path: linkPath + } + + return { + entries: [ + basic + ] + } + } + + const result = await resolve(link.Hash, name, linkPath, [], depth + 1, blockstore, options) + + return { + entries: [ + result.entry + ].filter(Boolean) + } } else { // descend into subshard - const block = await blockstore.get(link.Hash, options) + const block = await toBuffer(blockstore.get(link.Hash, options)) node = decode(block) options.onProgress?.(new CustomProgressEvent('unixfs:exporter:walk:hamt-sharded-directory', { cid: link.Hash })) - return { entries: listDirectory(node, path, resolve, depth, blockstore, options) } + return { + entries: listDirectory(node, path, resolve, depth, blockstore, options) + } } } }), diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/index.ts b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/index.ts index 4c5983e4..0e2250f3 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/index.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/index.ts @@ -1,11 +1,13 @@ import { decode } from '@ipld/dag-pb' import { UnixFS } from 'ipfs-unixfs' +import toBuffer from 'it-to-buffer' import { NotFoundError, NotUnixFSError } from '../../errors.js' import findShardCid from '../../utils/find-cid-in-shard.js' +import { isBasicExporterOptions } from '../../utils/is-basic-exporter-options.ts' import contentDirectory from './content/directory.js' import contentFile from './content/file.js' import contentHamtShardedDirectory from './content/hamt-sharded-directory.js' -import type { Resolver, UnixfsV1Resolver } from '../../index.js' +import type { Resolver, UnixFSBasicEntry, UnixfsV1Resolver } from '../../index.js' import type { PBNode } from '@ipld/dag-pb' import type { CID } from 'multiformats/cid' @@ -30,7 +32,19 @@ const contentExporters: Record = { // @ts-expect-error types are wrong const unixFsResolver: Resolver = async (cid, name, path, toResolve, resolve, depth, blockstore, options) => { - const block = await blockstore.get(cid, options) + if (isBasicExporterOptions(options) && toResolve.length === 0) { + const basic: UnixFSBasicEntry = { + cid, + name, + path + } + + return { + entry: basic + } + } + + const block = await toBuffer(blockstore.get(cid, options)) const node = decode(block) let unixfs let next diff --git a/packages/ipfs-unixfs-exporter/src/utils/find-cid-in-shard.ts b/packages/ipfs-unixfs-exporter/src/utils/find-cid-in-shard.ts index d604ff28..c88269c1 100644 --- a/packages/ipfs-unixfs-exporter/src/utils/find-cid-in-shard.ts +++ b/packages/ipfs-unixfs-exporter/src/utils/find-cid-in-shard.ts @@ -2,6 +2,7 @@ import { decode } from '@ipld/dag-pb' import { murmur3128 } from '@multiformats/murmur3' import { Bucket, createHAMT } from 'hamt-sharding' import { UnixFS } from 'ipfs-unixfs' +import toBuffer from 'it-to-buffer' import { NotUnixFSError } from '../errors.js' import type { ExporterOptions, ShardTraversalContext, ReadableStorage } from '../index.js' import type { PBLink, PBNode } from '@ipld/dag-pb' @@ -142,7 +143,7 @@ const findShardCid = async (node: PBNode, name: string, blockstore: ReadableStor context.hamtDepth++ - const block = await blockstore.get(link.Hash, options) + const block = await toBuffer(blockstore.get(link.Hash, options)) node = decode(block) return findShardCid(node, name, blockstore, context, options) diff --git a/packages/ipfs-unixfs-exporter/src/utils/is-basic-exporter-options.ts b/packages/ipfs-unixfs-exporter/src/utils/is-basic-exporter-options.ts new file mode 100644 index 00000000..95190ea5 --- /dev/null +++ b/packages/ipfs-unixfs-exporter/src/utils/is-basic-exporter-options.ts @@ -0,0 +1,5 @@ +import type { BasicExporterOptions } from '../index.js' + +export function isBasicExporterOptions (obj?: any): obj is BasicExporterOptions { + return obj?.extended === false +} diff --git a/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts b/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts index dd2354c5..35af64b4 100644 --- a/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts +++ b/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts @@ -8,6 +8,7 @@ import { importer } from 'ipfs-unixfs-importer' import all from 'it-all' import randomBytes from 'it-buffer-stream' import last from 'it-last' +import toBuffer from 'it-to-buffer' import { CID } from 'multiformats/cid' import { sha256 } from 'multiformats/hashes/sha2' import { concat as uint8ArrayConcat } from 'uint8arrays/concat' @@ -83,7 +84,7 @@ describe('exporter sharded', function () { files[imported.path].cid = imported.cid }) - const encodedBlock = await block.get(dirCid) + const encodedBlock = await toBuffer(block.get(dirCid)) const dir = dagPb.decode(encodedBlock) if (dir.Data == null) { throw Error('PBNode Data undefined') @@ -363,4 +364,88 @@ describe('exporter sharded', function () { content: file?.node }]).to.deep.equal(files) }) + + it('exports basic sharded directory', async () => { + const files: Record = {} + + // needs to result in a block that is larger than SHARD_SPLIT_THRESHOLD bytes + for (let i = 0; i < 100; i++) { + files[`file-${Math.random()}.txt`] = { + content: uint8ArrayConcat(await all(randomBytes(100))) + } + } + + const imported = await all(importer(Object.keys(files).map(path => ({ + path, + content: asAsyncIterable(files[path].content) + })), block, { + wrapWithDirectory: true, + shardSplitThresholdBytes: SHARD_SPLIT_THRESHOLD, + rawLeaves: false + })) + + const dirCid = imported.pop()?.cid + + if (dirCid == null) { + throw new Error('No directory CID found') + } + + const exported = await exporter(dirCid, block) + const dirFiles = await all(exported.content()) + + // delete shard contents + for (const entry of dirFiles) { + await block.delete(entry.cid) + } + + // list the contents again, this time just the basic version + const basicDirFiles = await all(exported.content({ + extended: false + })) + expect(basicDirFiles.length).to.equal(dirFiles.length) + + for (let i = 0; i < basicDirFiles.length; i++) { + const dirFile = basicDirFiles[i] + + expect(dirFile).to.have.property('name') + expect(dirFile).to.have.property('path') + expect(dirFile).to.have.property('cid') + + // should fail because we have deleted this block + await expect(exporter(dirFile.cid, block)).to.eventually.be.rejected() + } + }) + + it('exports basic file from sharded directory', async () => { + const files: Record = {} + + // needs to result in a block that is larger than SHARD_SPLIT_THRESHOLD bytes + for (let i = 0; i < 100; i++) { + files[`file-${Math.random()}.txt`] = { + content: uint8ArrayConcat(await all(randomBytes(100))) + } + } + + const imported = await all(importer(Object.keys(files).map(path => ({ + path, + content: asAsyncIterable(files[path].content) + })), block, { + wrapWithDirectory: true, + shardSplitThresholdBytes: SHARD_SPLIT_THRESHOLD, + rawLeaves: false + })) + + const file = imported[0] + const dir = imported[imported.length - 1] + + const basicfile = await exporter(`/ipfs/${dir.cid}/${file.path}`, block, { + extended: false + }) + + expect(basicfile).to.have.property('name', file.path) + expect(basicfile).to.have.property('path', `${dir.cid}/${file.path}`) + expect(basicfile).to.have.deep.property('cid', file.cid) + expect(basicfile).to.not.have.property('unixfs') + expect(basicfile).to.not.have.property('content') + }) }) diff --git a/packages/ipfs-unixfs-exporter/test/exporter.spec.ts b/packages/ipfs-unixfs-exporter/test/exporter.spec.ts index 67326ec4..f915b6c1 100644 --- a/packages/ipfs-unixfs-exporter/test/exporter.spec.ts +++ b/packages/ipfs-unixfs-exporter/test/exporter.spec.ts @@ -150,7 +150,7 @@ describe('exporter', () => { it('ensure hash inputs are sanitized', async () => { const result = await dagPut() - const encodedBlock = await block.get(result.cid) + const encodedBlock = await toBuffer(block.get(result.cid)) const node = dagPb.decode(encodedBlock) if (node.Data == null) { throw new Error('PBNode Data undefined') @@ -212,7 +212,7 @@ describe('exporter', () => { content: uint8ArrayConcat(await all(randomBytes(100))) }) - const encodedBlock = await block.get(result.cid) + const encodedBlock = await toBuffer(block.get(result.cid)) const node = dagPb.decode(encodedBlock) if (node.Data == null) { throw new Error('PBNode Data undefined') @@ -339,10 +339,10 @@ describe('exporter', () => { // @ts-expect-error incomplete implementation const blockStore: Blockstore = { ...block, - async get (cid: CID) { + async * get (cid: CID) { await delay(Math.random() * 10) - return block.get(cid) + yield * block.get(cid) } } @@ -1289,9 +1289,10 @@ describe('exporter', () => { // regular test IPLD is offline-only, we need to mimic what happens when // we try to get a block from the network const customBlock = { - get: async (cid: CID, options: { signal: AbortSignal }) => { + // eslint-disable-next-line require-yield + get: async function * (cid: CID, options: { signal: AbortSignal }) { // promise will never resolve, so reject it when the abort signal is sent - return new Promise((resolve, reject) => { + await new Promise((resolve, reject) => { options.signal.addEventListener('abort', () => { reject(new Error(message)) }) @@ -1299,7 +1300,6 @@ describe('exporter', () => { } } - // @ts-expect-error ipld implementation incomplete await expect(exporter(cid, customBlock, { signal: abortController.signal })).to.eventually.be.rejectedWith(message) @@ -1397,13 +1397,13 @@ describe('exporter', () => { throw new Error('Nothing imported') } - const node = dagPb.decode(await block.get(imported.cid)) + const node = dagPb.decode(await toBuffer(block.get(imported.cid))) expect(node.Links).to.have.lengthOf(2, 'imported node had too many children') - const child1 = dagPb.decode(await block.get(node.Links[0].Hash)) + const child1 = dagPb.decode(await toBuffer(block.get(node.Links[0].Hash))) expect(child1.Links).to.have.lengthOf(2, 'layer 1 node had too many children') - const child2 = dagPb.decode(await block.get(node.Links[1].Hash)) + const child2 = dagPb.decode(await toBuffer(block.get(node.Links[1].Hash))) expect(child2.Links).to.have.lengthOf(2, 'layer 1 node had too many children') // should be raw nodes @@ -1468,7 +1468,7 @@ describe('exporter', () => { throw new Error('Nothing imported') } - const node = dagPb.decode(await block.get(imported.cid)) + const node = dagPb.decode(await toBuffer(block.get(imported.cid))) expect(node.Links).to.have.lengthOf(entries, 'imported node had too many children') for (const link of node.Links) { @@ -1491,7 +1491,7 @@ describe('exporter', () => { const actualInvocations: string[] = [] - block.get = async (cid) => { + block.get = async function * (cid) { actualInvocations.push(`${cid.toString()}-start`) // introduce a small delay - if running in parallel actualInvocations will @@ -1503,7 +1503,7 @@ describe('exporter', () => { actualInvocations.push(`${cid.toString()}-end`) - return originalGet(cid) + yield * originalGet(cid) } const blockReadSpy = Sinon.spy(block, 'get') @@ -1539,7 +1539,7 @@ describe('exporter', () => { throw new Error('Nothing imported') } - const node = dagPb.decode(await block.get(imported.cid)) + const node = dagPb.decode(await toBuffer(block.get(imported.cid))) const data = UnixFS.unmarshal(node.Data ?? new Uint8Array(0)) expect(data.type).to.equal('hamt-sharded-directory') @@ -1551,7 +1551,7 @@ describe('exporter', () => { children.push(link.Hash) if (link.Hash.code === dagPb.code) { - const buf = await block.get(link.Hash) + const buf = await toBuffer(block.get(link.Hash)) const childNode = dagPb.decode(buf) children.push(...(await collectCIDs(childNode))) @@ -1578,7 +1578,7 @@ describe('exporter', () => { const actualInvocations: string[] = [] - block.get = async (cid) => { + block.get = async function * (cid) { actualInvocations.push(`${cid.toString()}-start`) // introduce a small delay - if running in parallel actualInvocations will @@ -1590,7 +1590,7 @@ describe('exporter', () => { actualInvocations.push(`${cid.toString()}-end`) - return originalGet(cid) + yield * originalGet(cid) } const blockReadSpy = Sinon.spy(block, 'get') @@ -1605,4 +1605,136 @@ describe('exporter', () => { expect(actualInvocations).to.deep.equal(expectedInvocations) }) + + it('exports basic directory contents', async () => { + const files: Record = {} + + for (let i = 0; i < 10; i++) { + files[`file-${Math.random()}.txt`] = { + content: uint8ArrayConcat(await all(randomBytes(100))) + } + } + + const imported = await all(importer(Object.keys(files).map(path => ({ + path, + content: asAsyncIterable(files[path].content) + })), block, { + wrapWithDirectory: true, + rawLeaves: false + })) + + const dirCid = imported.pop()?.cid + + if (dirCid == null) { + throw new Error('No directory CID found') + } + + const exported = await exporter(dirCid, block) + const dirFiles = await all(exported.content()) + + // delete shard contents + for (const entry of dirFiles) { + await block.delete(entry.cid) + } + + // list the contents again, this time just the basic version + const basicDirFiles = await all(exported.content({ + extended: false + })) + expect(basicDirFiles.length).to.equal(dirFiles.length) + + for (let i = 0; i < basicDirFiles.length; i++) { + const dirFile = basicDirFiles[i] + + expect(dirFile).to.have.property('name') + expect(dirFile).to.have.property('path') + expect(dirFile).to.have.property('cid') + + // should fail because we have deleted this block + await expect(exporter(dirFile.cid, block)).to.eventually.be.rejected() + } + }) + + it('exports basic file', async () => { + const imported = await all(importer([{ + content: uint8ArrayFromString('hello') + }], block, { + rawLeaves: false + })) + + const regularFile = await exporter(imported[0].cid, block) + expect(regularFile).to.have.property('unixfs') + + const basicFile = await exporter(imported[0].cid, block, { + extended: false + }) + + expect(basicFile).to.have.property('name') + expect(basicFile).to.have.property('path') + expect(basicFile).to.have.property('cid') + expect(basicFile).to.not.have.property('unixfs') + }) + + it('exports basic directory', async () => { + const files: Record = {} + + for (let i = 0; i < 10; i++) { + files[`file-${Math.random()}.txt`] = { + content: uint8ArrayConcat(await all(randomBytes(100))) + } + } + + const imported = await all(importer(Object.keys(files).map(path => ({ + path, + content: asAsyncIterable(files[path].content) + })), block, { + wrapWithDirectory: true, + rawLeaves: false + })) + + const dirCid = imported.pop()?.cid + + if (dirCid == null) { + throw new Error('No directory CID found') + } + + const basicDir = await exporter(dirCid, block, { + extended: false + }) + + expect(basicDir).to.have.property('name') + expect(basicDir).to.have.property('path') + expect(basicDir).to.have.property('cid') + expect(basicDir).to.not.have.property('unixfs') + expect(basicDir).to.not.have.property('content') + }) + + it('exports basic file from directory', async () => { + const files: Record = { + 'file.txt': { + content: uint8ArrayConcat(await all(randomBytes(100))) + } + } + + const imported = await all(importer(Object.keys(files).map(path => ({ + path, + content: asAsyncIterable(files[path].content) + })), block, { + wrapWithDirectory: true, + rawLeaves: false + })) + + const file = imported[0] + const dir = imported[imported.length - 1] + + const basicfile = await exporter(`/ipfs/${dir.cid}/${file.path}`, block, { + extended: false + }) + + expect(basicfile).to.have.property('name', file.path) + expect(basicfile).to.have.property('path', `${dir.cid}/${file.path}`) + expect(basicfile).to.have.deep.property('cid', file.cid) + expect(basicfile).to.not.have.property('unixfs') + expect(basicfile).to.not.have.property('content') + }) }) diff --git a/packages/ipfs-unixfs-exporter/test/helpers/collect-leaf-cids.ts b/packages/ipfs-unixfs-exporter/test/helpers/collect-leaf-cids.ts index 564ead38..251e5b84 100644 --- a/packages/ipfs-unixfs-exporter/test/helpers/collect-leaf-cids.ts +++ b/packages/ipfs-unixfs-exporter/test/helpers/collect-leaf-cids.ts @@ -1,10 +1,11 @@ import * as dagPb from '@ipld/dag-pb' +import toBuffer from 'it-to-buffer' import type { Blockstore } from 'interface-blockstore' import type { CID } from 'multiformats/cid' export default function (cid: CID, blockstore: Blockstore): AsyncGenerator<{ node: Uint8Array | dagPb.PBNode, cid: CID }, void, undefined> { async function * traverse (cid: CID): AsyncGenerator<{ node: dagPb.PBNode, cid: CID }, void, unknown> { - const block = await blockstore.get(cid) + const block = await toBuffer(blockstore.get(cid)) const node = dagPb.decode(block) if (node instanceof Uint8Array || (node.Links.length === 0)) { diff --git a/packages/ipfs-unixfs-exporter/test/importer.spec.ts b/packages/ipfs-unixfs-exporter/test/importer.spec.ts index 2684468b..9db2fd64 100644 --- a/packages/ipfs-unixfs-exporter/test/importer.spec.ts +++ b/packages/ipfs-unixfs-exporter/test/importer.spec.ts @@ -11,6 +11,7 @@ import { balanced, flat, trickle } from 'ipfs-unixfs-importer/layout' import all from 'it-all' import first from 'it-first' import last from 'it-last' +import toBuffer from 'it-to-buffer' // @ts-expect-error https://github.com/schnittstabil/merge-options/pull/28 import extend from 'merge-options' import { base58btc } from 'multiformats/bases/base58' @@ -203,7 +204,7 @@ const checkLeafNodeTypes = async (blockstore: Blockstore, options: Partial blockstore.get(link.Hash)) + node.Links.map(async link => toBuffer(blockstore.get(link.Hash))) ) linkedBlocks.forEach(bytes => { @@ -232,7 +233,7 @@ const checkNodeLinks = async (blockstore: Blockstore, options: Partial +- # License diff --git a/packages/ipfs-unixfs/package.json b/packages/ipfs-unixfs/package.json index fbace349..723aaf8a 100644 --- a/packages/ipfs-unixfs/package.json +++ b/packages/ipfs-unixfs/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-unixfs", - "version": "11.2.5", + "version": "12.0.0", "description": "JavaScript implementation of IPFS' unixfs (a Unix FileSystem representation on top of a MerkleDAG)", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/main/packages/ipfs-unixfs#readme",