diff --git a/.github/workflows/npm-publish-react-native-executorch-webrtc.yml b/.github/workflows/npm-publish-react-native-executorch-webrtc.yml new file mode 100644 index 0000000000..ee20b67e81 --- /dev/null +++ b/.github/workflows/npm-publish-react-native-executorch-webrtc.yml @@ -0,0 +1,119 @@ +name: NPM publish react-native-executorch-webrtc + +on: + workflow_dispatch: + inputs: + latest-build: + description: 'Whether to publish as a latest build' + required: true + type: boolean + +permissions: + id-token: write + contents: read + +concurrency: + group: 'npm-react-native-executorch-webrtc-build' + cancel-in-progress: false + +jobs: + build: + if: github.repository == 'software-mansion/react-native-executorch' + runs-on: ubuntu-latest + environment: deployment + permissions: + contents: read + id-token: write + env: + PACKAGE_DIR: packages/react-native-executorch-webrtc + PACKAGE_VERSION: PLACEHOLDER + PACKAGE_NAME: PLACEHOLDER + TAG: PLACEHOLDER + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Setup Node + uses: actions/setup-node@v6 + with: + node-version: 24 + cache: 'yarn' + registry-url: https://registry.npmjs.org/ + + - name: Update NPM + run: npm install -g npm@latest + + - name: Determine version + working-directory: ${{ env.PACKAGE_DIR }} + run: | + VERSION=$(jq -r .version package.json) + echo "PACKAGE_VERSION=$VERSION" >> $GITHUB_ENV + + - name: Assert PACKAGE_VERSION + if: ${{ env.PACKAGE_VERSION == 'PLACEHOLDER' }} + run: exit 1 # this should never happen + + - name: Install monorepo dependencies + run: yarn install --immutable + + - name: Set tag + run: | + if [[ "${{ inputs.latest-build }}" != "true" ]]; then + echo "TAG=executorch-nightly" >> $GITHUB_ENV + else + echo "TAG=latest" >> $GITHUB_ENV + fi + + - name: Assert tag + if: ${{ env.TAG == 'PLACEHOLDER' }} + run: exit 1 # this should never happen + + - name: Set nightly version + if: ${{ !inputs.latest-build }} + working-directory: ${{ env.PACKAGE_DIR }} + run: | + VERSION=${{ env.PACKAGE_VERSION }} + IFS='.' read -r MAJOR MINOR PATCH <<< "$VERSION" + GIT_COMMIT=$(git rev-parse HEAD) + DATE=$(date +%Y%m%d) + NIGHTLY_UNIQUE_NAME="${GIT_COMMIT:0:7}-$DATE" + sed -i "3s/.*/ \"version\": \"$MAJOR.$MINOR.$PATCH-nightly-$NIGHTLY_UNIQUE_NAME\",/" package.json + + - name: Build core package + working-directory: packages/react-native-executorch + run: yarn prepare + + - name: Build package + working-directory: ${{ env.PACKAGE_DIR }} + run: yarn prepare + + - name: Pack package + working-directory: ${{ env.PACKAGE_DIR }} + run: npm pack + + - name: Restore version + if: ${{ !inputs.latest-build }} + working-directory: ${{ env.PACKAGE_DIR }} + run: | + VERSION=${{ env.PACKAGE_VERSION }} + sed -i "3s/.*/ \"version\": \"$VERSION\",/" package.json + + - name: Add package name to env + working-directory: ${{ env.PACKAGE_DIR }} + run: echo "PACKAGE_NAME=$(ls -l | egrep -o "react-native-executorch-webrtc-(.*)(=?\.tgz)")" >> $GITHUB_ENV + + - name: Assert package name + if: ${{ env.PACKAGE_NAME == 'PLACEHOLDER' }} + run: exit 1 # this should never happen + + - name: Upload package to GitHub + uses: actions/upload-artifact@v4 + with: + name: ${{ env.PACKAGE_NAME }} + path: ${{ env.PACKAGE_DIR }}/${{ env.PACKAGE_NAME }} + + - name: Move package to monorepo root + run: mv ${{ env.PACKAGE_DIR }}/${{ env.PACKAGE_NAME }} . + + - name: Publish package to npm + run: npm publish $PACKAGE_NAME --tag ${{ env.TAG }} --provenance --access public diff --git a/docs/docs/05-utilities/01-webrtc-integration.md b/docs/docs/05-utilities/01-webrtc-integration.md new file mode 100644 index 0000000000..a759709d80 --- /dev/null +++ b/docs/docs/05-utilities/01-webrtc-integration.md @@ -0,0 +1,118 @@ +--- +title: Fishjam Usage +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +:::danger +This integration is currently in beta. +::: + +## Overview + +Starting from v0.9.0, you can use `react-native-executorch` powered background blur integration in your [Fishjam](https://fishjam.io) applications. The package `react-native-executorch-webrtc` exposes a hook, which returns a middleware for your camera streams. We plan to extend this to other models, such as text to speech, speech to text, or other vision models in the future. + +## Installation + +Install the package with your package manager of choice. Make sure to also have `react-native-executorch` and a resource fetcher adapter installed (see [Getting Started](../01-fundamentals/01-getting-started.md)). + + + + + ```bash + npm install react-native-executorch-webrtc + ``` + + + + + ```bash + pnpm install react-native-executorch-webrtc + ``` + + + + + ```bash + yarn add react-native-executorch-webrtc + ``` + + + + +The following peer dependencies must also be installed in your app: + +- `@fishjam-cloud/react-native-client` +- `@fishjam-cloud/react-native-webrtc` +- `react-native-executorch` + +## Usage + +The integration is built around the `selfie_segmentation` model that we expose through the [Model Registry](./model-registry.md). It's the only model we currently support and tune for — the blur pipeline expects its specific input shape and output classes, so other segmentation models will not work correctly. + +Use `ResourceFetcher` together with `models.semantic_segmentation.selfie_segmentation().modelSource` to download (and cache) the model, then pass the resulting path to `useBackgroundBlur`. The returned `blurMiddleware` plugs into Fishjam's `cameraTrackMiddleware`. + +```tsx +import { useEffect, useState } from 'react'; +import { Button, Text } from 'react-native'; +import { models, ResourceFetcher } from 'react-native-executorch'; +import { useBackgroundBlur } from 'react-native-executorch-webrtc'; +import { useCamera } from '@fishjam-cloud/react-native-client'; + +function VideoCall() { + const [modelUri, setModelUri] = useState(null); + + useEffect(() => { + ResourceFetcher.fetch( + () => {}, + models.semantic_segmentation.selfie_segmentation().modelSource + ).then((paths) => paths?.[0] && setModelUri(paths[0])); + }, []); + + // Wait for the model to be available before mounting the hook — + // useBackgroundBlur expects a real path, not an empty string. + if (!modelUri) { + return Downloading model…; + } + + return ; +} + +function VideoCallWithBlur({ modelUri }: { modelUri: string }) { + const [blurEnabled, setBlurEnabled] = useState(true); + + const { blurMiddleware } = useBackgroundBlur({ + modelUri, + blurRadius: 15, + }); + + useCamera({ + cameraTrackMiddleware: blurEnabled ? blurMiddleware : undefined, + }); + + return ( +