Merge pull request #174 from ace-step/add_sde_sampler_support

add stable audio small pingpong sampler support

README.md

@@ -30,6 +30,8 @@ Rather than building yet another end-to-end text-to-music pipeline, our vision i
 
 
 ## 📢 News and Updates
+- 🎮 2025.05.14: Add `Stable Audio Open Small` sampler `pingpong`. Use SDE to achieve better music consistency and quality, including lyric alignment and style alignment. Use a better method to re-implement `Audio2Audio`
+
 - 🎤 2025.05.12: Release [RapMachine](https://huggingface.co/ACE-Step/ACE-Step-v1-chinese-rap-LoRA) and fix lora training issues
     - See [ZH_RAP_LORA.md](./ZH_RAP_LORA.md) for details. Audio Examples: https://ace-step.github.io/#RapMachine
     - See [TRAIN_INSTRUCTION.md](./TRAIN_INSTRUCTION.md) for detailed training instructions.

acestep/pipeline_ace_step.py

@@ -26,6 +26,9 @@ from acestep.schedulers.scheduling_flow_match_euler_discrete import (
 from acestep.schedulers.scheduling_flow_match_heun_discrete import (
     FlowMatchHeunDiscreteScheduler,
 )
+from acestep.schedulers.scheduling_flow_match_pingpong import (
+    FlowMatchPingPongScheduler,
+)
 from diffusers.pipelines.stable_diffusion_3.pipeline_stable_diffusion_3 import (
     retrieve_timesteps,
 )
@@ -770,6 +773,7 @@ class ACEStepPipeline:
         n_min=0,
         n_max=1.0,
         n_avg=1,
+        scheduler_type="euler",
     ):
 
         do_classifier_free_guidance = True
@@ -897,10 +901,18 @@ class ACEStepPipeline:
                         Vt_tar - Vt_src
                     )  # - (hfg-1)*( x_src))
 
-                # propagate direct ODE
                 zt_edit = zt_edit.to(torch.float32)
-                zt_edit = zt_edit + (t_im1 - t_i) * V_delta_avg
+                if scheduler_type != "pingpong":
-                zt_edit = zt_edit.to(V_delta_avg.dtype)
+                    # propagate direct ODE
+                    zt_edit = zt_edit.to(torch.float32)
+                    zt_edit = zt_edit + (t_im1 - t_i) * V_delta_avg
+                    zt_edit = zt_edit.to(V_delta_avg.dtype)
+                else:
+                    # propagate pingpong SDE
+                    zt_edit_denoised = zt_edit - t_i * V_delta_avg
+                    noise = torch.empty_like(zt_edit).normal_(generator=random_generators[0] if random_generators else None)
+                    prev_sample = (1 - t_im1) * zt_edit_denoised + t_im1 * noise
+
             else:  # i >= T_steps-n_min # regular sampling for last n_min steps
                 if i == n_max:
                     fwd_noise = randn_tensor(
@@ -938,9 +950,15 @@ class ACEStepPipeline:
 
                 dtype = Vt_tar.dtype
                 xt_tar = xt_tar.to(torch.float32)
-                prev_sample = xt_tar + (t_im1 - t_i) * Vt_tar
+                if scheduler_type != "pingpong":
-                prev_sample = prev_sample.to(dtype)
+                    prev_sample = xt_tar + (t_im1 - t_i) * Vt_tar
-                xt_tar = prev_sample
+                    prev_sample = prev_sample.to(dtype)
+                    xt_tar = prev_sample
+                else:
+                    prev_sample = xt_tar - t_i * Vt_tar
+                    noise = torch.empty_like(zt_edit).normal_(generator=random_generators[0] if random_generators else None)
+                    prev_sample = (1 - t_im1) * prev_sample + t_im1 * noise
+                    xt_tar = prev_sample
 
         target_latents = zt_edit if xt_tar is None else xt_tar
         return target_latents
@@ -948,23 +966,43 @@ class ACEStepPipeline:
     def add_latents_noise(
         self,
         gt_latents,
-        variance,
+        sigma_max,
         noise,
-        scheduler,
+        scheduler_type,
+        infer_steps,
     ):
 
         bsz = gt_latents.shape[0]
-        u = torch.tensor([variance] * bsz, dtype=gt_latents.dtype)
+        device = gt_latents.device
-        indices = (u * scheduler.config.num_train_timesteps).long()
+        noisy_image = gt_latents * (1 - sigma_max) + noise * sigma_max
-        timesteps = scheduler.timesteps.unsqueeze(1).to(gt_latents.dtype)
+        if scheduler_type == "euler":
-        indices = indices.to(timesteps.device).to(gt_latents.dtype).unsqueeze(1)
+            scheduler = FlowMatchEulerDiscreteScheduler(
-        nearest_idx = torch.argmin(torch.cdist(indices, timesteps), dim=1)
+                num_train_timesteps=1000,
-        sigma = scheduler.sigmas[nearest_idx].flatten().to(gt_latents.device).to(gt_latents.dtype)
+                shift=3.0,
-        while len(sigma.shape) < gt_latents.ndim:
+                sigma_max=sigma_max,
-            sigma = sigma.unsqueeze(-1)
+            )
-        noisy_image = sigma * noise + (1.0 - sigma) * gt_latents
+        elif scheduler_type == "heun":
-        init_timestep = indices[0]
+            scheduler = FlowMatchHeunDiscreteScheduler(
-        return noisy_image, init_timestep
+                num_train_timesteps=1000,
+                shift=3.0,
+                sigma_max=sigma_max,
+            )
+        elif scheduler_type == "pingpong":
+            scheduler = FlowMatchPingPongScheduler(
+                num_train_timesteps=1000,
+                shift=3.0,
+                sigma_max=sigma_max
+            )
+
+        infer_steps = int(sigma_max * infer_steps)
+        timesteps, num_inference_steps = retrieve_timesteps(
+            scheduler,
+            num_inference_steps=infer_steps,
+            device=device,
+            timesteps=None,
+        )
+        logger.info(f"{scheduler.sigma_min=} {scheduler.sigma_max=} {timesteps=} {num_inference_steps=}")
+        return noisy_image, timesteps, scheduler, num_inference_steps
 
     @cpu_offload("ace_step_transformer")
     @torch.no_grad()
@@ -1043,6 +1081,11 @@ class ACEStepPipeline:
                 num_train_timesteps=1000,
                 shift=3.0,
             )
+        elif scheduler_type == "pingpong":
+            scheduler = FlowMatchPingPongScheduler(
+                num_train_timesteps=1000,
+                shift=3.0,
+            )
 
         frame_length = int(duration * 44100 / 512 / 8)
         if src_latents is not None:
@@ -1209,9 +1252,17 @@ class ACEStepPipeline:
                 zt_edit = x0.clone()
                 z0 = target_latents
 
-        init_timestep = 1000
         if audio2audio_enable and ref_latents is not None:
-            target_latents, init_timestep = self.add_latents_noise(gt_latents=ref_latents, variance=(1-ref_audio_strength), noise=target_latents, scheduler=scheduler)
+            logger.info(
+                f"audio2audio_enable: {audio2audio_enable}, ref_latents: {ref_latents.shape}"
+            )
+            target_latents, timesteps, scheduler, num_inference_steps = self.add_latents_noise(
+                gt_latents=ref_latents,
+                sigma_max=(1-ref_audio_strength),
+                noise=target_latents,
+                scheduler_type=scheduler_type,
+                infer_steps=infer_steps,
+            )
 
         attention_mask = torch.ones(bsz, frame_length, device=device, dtype=dtype)
 
@@ -1334,8 +1385,6 @@ class ACEStepPipeline:
             return sample
 
         for i, t in tqdm(enumerate(timesteps), total=num_inference_steps):
-            if t > init_timestep:
-                continue
 
             if is_repaint:
                 if i < n_min:
@@ -1483,6 +1532,7 @@ class ACEStepPipeline:
                     sample=target_latents,
                     return_dict=False,
                     omega=omega_scale,
+                    generator=random_generators[0],
                 )[0]
 
         if is_extend:
@@ -1773,6 +1823,7 @@ class ACEStepPipeline:
                 n_min=edit_n_min,
                 n_max=edit_n_max,
                 n_avg=edit_n_avg,
+                scheduler_type=scheduler_type,
             )
         else:
             target_latents = self.text2music_diffusion_process(

acestep/schedulers/scheduling_flow_match_euler_discrete.py

@@ -71,9 +71,10 @@ class FlowMatchEulerDiscreteScheduler(SchedulerMixin, ConfigMixin):
         max_shift: Optional[float] = 1.15,
         base_image_seq_len: Optional[int] = 256,
         max_image_seq_len: Optional[int] = 4096,
+        sigma_max: Optional[float] = 1.0,
     ):
         timesteps = np.linspace(
-            1, num_train_timesteps, num_train_timesteps, dtype=np.float32
+            1.0, sigma_max*num_train_timesteps, num_train_timesteps, dtype=np.float32
         )[::-1].copy()
         timesteps = torch.from_numpy(timesteps).to(dtype=torch.float32)
 

acestep/schedulers/scheduling_flow_match_heun_discrete.py

@@ -66,9 +66,10 @@ class FlowMatchHeunDiscreteScheduler(SchedulerMixin, ConfigMixin):
         self,
         num_train_timesteps: int = 1000,
         shift: float = 1.0,
+        sigma_max: Optional[float] = 1.0,
     ):
         timesteps = np.linspace(
-            1, num_train_timesteps, num_train_timesteps, dtype=np.float32
+            1.0, sigma_max*num_train_timesteps, num_train_timesteps, dtype=np.float32
         )[::-1].copy()
         timesteps = torch.from_numpy(timesteps).to(dtype=torch.float32)
 

acestep/schedulers/scheduling_flow_match_pingpong.py

@@ -0,0 +1,343 @@
+# Copyright 2024 Stability AI, Katherine Crowson and The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+from dataclasses import dataclass
+from typing import List, Optional, Tuple, Union
+
+import numpy as np
+import torch
+
+from diffusers.configuration_utils import ConfigMixin, register_to_config
+from diffusers.utils import BaseOutput, logging
+from diffusers.schedulers.scheduling_utils import SchedulerMixin
+
+
+logger = logging.get_logger(__name__)  # pylint: disable=invalid-name
+
+
+@dataclass
+class FlowMatchPingPongSchedulerOutput(BaseOutput):
+    """
+    Output class for the scheduler's `step` function output.
+
+    Args:
+        prev_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images):
+            Computed sample `(x_{t-1})` of previous timestep. `prev_sample` should be used as next model input in the
+            denoising loop.
+    """
+
+    prev_sample: torch.FloatTensor
+
+
+class FlowMatchPingPongScheduler(SchedulerMixin, ConfigMixin):
+    """
+    PingPong scheduler.
+
+    This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic
+    methods the library implements for all schedulers such as loading and saving.
+
+    Args:
+        num_train_timesteps (`int`, defaults to 1000):
+            The number of diffusion steps to train the model.
+        timestep_spacing (`str`, defaults to `"linspace"`):
+            The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and
+            Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information.
+        shift (`float`, defaults to 1.0):
+            The shift value for the timestep schedule.
+    """
+
+    _compatibles = []
+    order = 1
+
+    @register_to_config
+    def __init__(
+        self,
+        num_train_timesteps: int = 1000,
+        shift: float = 1.0,
+        use_dynamic_shifting=False,
+        base_shift: Optional[float] = 0.5,
+        max_shift: Optional[float] = 1.15,
+        base_image_seq_len: Optional[int] = 256,
+        max_image_seq_len: Optional[int] = 4096,
+        sigma_max: Optional[float] = 1.0,
+    ):
+        timesteps = np.linspace(
+            1, sigma_max*num_train_timesteps, num_train_timesteps, dtype=np.float32
+        )[::-1].copy()
+        timesteps = torch.from_numpy(timesteps).to(dtype=torch.float32)
+
+        sigmas = timesteps / num_train_timesteps
+        if not use_dynamic_shifting:
+            # when use_dynamic_shifting is True, we apply the timestep shifting on the fly based on the image resolution
+            sigmas = shift * sigmas / (1 + (shift - 1) * sigmas)
+
+        self.timesteps = sigmas * num_train_timesteps
+
+        self._step_index = None
+        self._begin_index = None
+
+        self.sigmas = sigmas.to("cpu")  # to avoid too much CPU/GPU communication
+        self.sigma_min = self.sigmas[-1].item()
+        self.sigma_max = self.sigmas[0].item()
+
+    @property
+    def step_index(self):
+        """
+        The index counter for current timestep. It will increase 1 after each scheduler step.
+        """
+        return self._step_index
+
+    @property
+    def begin_index(self):
+        """
+        The index for the first timestep. It should be set from pipeline with `set_begin_index` method.
+        """
+        return self._begin_index
+
+    # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.set_begin_index
+    def set_begin_index(self, begin_index: int = 0):
+        """
+        Sets the begin index for the scheduler. This function should be run from pipeline before the inference.
+
+        Args:
+            begin_index (`int`):
+                The begin index for the scheduler.
+        """
+        self._begin_index = begin_index
+
+    def scale_noise(
+        self,
+        sample: torch.FloatTensor,
+        timestep: Union[float, torch.FloatTensor],
+        noise: Optional[torch.FloatTensor] = None,
+    ) -> torch.FloatTensor:
+        """
+        Forward process in flow-matching
+
+        Args:
+            sample (`torch.FloatTensor`):
+                The input sample.
+            timestep (`int`, *optional*):
+                The current timestep in the diffusion chain.
+
+        Returns:
+            `torch.FloatTensor`:
+                A scaled input sample.
+        """
+        # Make sure sigmas and timesteps have the same device and dtype as original_samples
+        sigmas = self.sigmas.to(device=sample.device, dtype=sample.dtype)
+
+        if sample.device.type == "mps" and torch.is_floating_point(timestep):
+            # mps does not support float64
+            schedule_timesteps = self.timesteps.to(sample.device, dtype=torch.float32)
+            timestep = timestep.to(sample.device, dtype=torch.float32)
+        else:
+            schedule_timesteps = self.timesteps.to(sample.device)
+            timestep = timestep.to(sample.device)
+
+        # self.begin_index is None when scheduler is used for training, or pipeline does not implement set_begin_index
+        if self.begin_index is None:
+            step_indices = [
+                self.index_for_timestep(t, schedule_timesteps) for t in timestep
+            ]
+        elif self.step_index is not None:
+            # add_noise is called after first denoising step (for inpainting)
+            step_indices = [self.step_index] * timestep.shape[0]
+        else:
+            # add noise is called before first denoising step to create initial latent(img2img)
+            step_indices = [self.begin_index] * timestep.shape[0]
+
+        sigma = sigmas[step_indices].flatten()
+        while len(sigma.shape) < len(sample.shape):
+            sigma = sigma.unsqueeze(-1)
+
+        sample = sigma * noise + (1.0 - sigma) * sample
+
+        return sample
+
+    def _sigma_to_t(self, sigma):
+        return sigma * self.config.num_train_timesteps
+
+    def time_shift(self, mu: float, sigma: float, t: torch.Tensor):
+        return math.exp(mu) / (math.exp(mu) + (1 / t - 1) ** sigma)
+
+    def set_timesteps(
+        self,
+        num_inference_steps: int = None,
+        device: Union[str, torch.device] = None,
+        sigmas: Optional[List[float]] = None,
+        mu: Optional[float] = None,
+    ):
+        """
+        Sets the discrete timesteps used for the diffusion chain (to be run before inference).
+
+        Args:
+            num_inference_steps (`int`):
+                The number of diffusion steps used when generating samples with a pre-trained model.
+            device (`str` or `torch.device`, *optional*):
+                The device to which the timesteps should be moved to. If `None`, the timesteps are not moved.
+        """
+
+        if self.config.use_dynamic_shifting and mu is None:
+            raise ValueError(
+                " you have a pass a value for `mu` when `use_dynamic_shifting` is set to be `True`"
+            )
+
+        if sigmas is None:
+            self.num_inference_steps = num_inference_steps
+            timesteps = np.linspace(
+                self._sigma_to_t(self.sigma_max),
+                self._sigma_to_t(self.sigma_min),
+                num_inference_steps,
+            )
+
+            sigmas = timesteps / self.config.num_train_timesteps
+
+        if self.config.use_dynamic_shifting:
+            sigmas = self.time_shift(mu, 1.0, sigmas)
+        else:
+            sigmas = self.config.shift * sigmas / (1 + (self.config.shift - 1) * sigmas)
+
+        sigmas = torch.from_numpy(sigmas).to(dtype=torch.float32, device=device)
+        timesteps = sigmas * self.config.num_train_timesteps
+
+        self.timesteps = timesteps.to(device=device)
+        self.sigmas = torch.cat([sigmas, torch.zeros(1, device=sigmas.device)])
+
+        self._step_index = None
+        self._begin_index = None
+
+    def index_for_timestep(self, timestep, schedule_timesteps=None):
+        if schedule_timesteps is None:
+            schedule_timesteps = self.timesteps
+
+        indices = (schedule_timesteps == timestep).nonzero()
+
+        # The sigma index that is taken for the **very** first `step`
+        # is always the second index (or the last index if there is only 1)
+        # This way we can ensure we don't accidentally skip a sigma in
+        # case we start in the middle of the denoising schedule (e.g. for image-to-image)
+        pos = 1 if len(indices) > 1 else 0
+
+        return indices[pos].item()
+
+    def _init_step_index(self, timestep):
+        if self.begin_index is None:
+            if isinstance(timestep, torch.Tensor):
+                timestep = timestep.to(self.timesteps.device)
+            self._step_index = self.index_for_timestep(timestep)
+        else:
+            self._step_index = self._begin_index
+
+    def step(
+        self,
+        model_output: torch.FloatTensor,
+        timestep: Union[float, torch.FloatTensor],
+        sample: torch.FloatTensor,
+        s_churn: float = 0.0,
+        s_tmin: float = 0.0,
+        s_tmax: float = float("inf"),
+        s_noise: float = 1.0,
+        generator: Optional[torch.Generator] = None,
+        return_dict: bool = True,
+        omega: Union[float, np.array] = 0.0,
+    ) -> Union[FlowMatchPingPongSchedulerOutput, Tuple]:
+        """
+        Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion
+        process from the learned model outputs (most often the predicted noise).
+
+        Args:
+            model_output (`torch.FloatTensor`):
+                The direct output from learned diffusion model.
+            timestep (`float`):
+                The current discrete timestep in the diffusion chain.
+            sample (`torch.FloatTensor`):
+                A current instance of a sample created by the diffusion process.
+            s_churn (`float`):
+            s_tmin  (`float`):
+            s_tmax  (`float`):
+            s_noise (`float`, defaults to 1.0):
+                Scaling factor for noise added to the sample.
+            generator (`torch.Generator`, *optional*):
+                A random number generator.
+            return_dict (`bool`):
+                Whether or not to return a [`~schedulers.scheduling_euler_discrete.EulerDiscreteSchedulerOutput`] or
+                tuple.
+
+        Returns:
+            [`~schedulers.scheduling_euler_discrete.EulerDiscreteSchedulerOutput`] or `tuple`:
+                If return_dict is `True`, [`~schedulers.scheduling_euler_discrete.EulerDiscreteSchedulerOutput`] is
+                returned, otherwise a tuple is returned where the first element is the sample tensor.
+        """
+
+        def logistic_function(x, L=0.9, U=1.1, x_0=0.0, k=1):
+            # L = Lower bound
+            # U = Upper bound
+            # x_0 = Midpoint (x corresponding to y = 1.0)
+            # k = Steepness, can adjust based on preference
+
+            if isinstance(x, torch.Tensor):
+                device_ = x.device
+                x = x.to(torch.float).cpu().numpy()
+
+            new_x = L + (U - L) / (1 + np.exp(-k * (x - x_0)))
+
+            if isinstance(new_x, np.ndarray):
+                new_x = torch.from_numpy(new_x).to(device_)
+            return new_x
+
+        self.omega_bef_rescale = omega
+        omega = logistic_function(omega, k=0.1)
+        self.omega_aft_rescale = omega
+
+        if (
+            isinstance(timestep, int)
+            or isinstance(timestep, torch.IntTensor)
+            or isinstance(timestep, torch.LongTensor)
+        ):
+            raise ValueError(
+                (
+                    "Passing integer indices (e.g. from `enumerate(timesteps)`) as timesteps to"
+                    " `EulerDiscreteScheduler.step()` is not supported. Make sure to pass"
+                    " one of the `scheduler.timesteps` as a timestep."
+                ),
+            )
+
+        if self.step_index is None:
+            self._init_step_index(timestep)
+
+        # Upcast to avoid precision issues when computing prev_sample
+        sample = sample.to(torch.float32)
+
+        sigma = self.sigmas[self.step_index]
+        sigma_next = self.sigmas[self.step_index + 1]
+
+        denoised = sample - sigma * model_output
+        noise = torch.empty_like(sample).normal_(generator=generator)
+        prev_sample = (1 - sigma_next) * denoised + sigma_next * noise
+
+        # Cast sample back to model compatible dtype
+        prev_sample = prev_sample.to(model_output.dtype)
+
+        # upon completion increase step index by one
+        self._step_index += 1
+
+        if not return_dict:
+            return (prev_sample,)
+
+        return FlowMatchPingPongSchedulerOutput(prev_sample=prev_sample)
+
+    def __len__(self):
+        return self.config.num_train_timesteps

acestep/ui/components.py

@@ -97,8 +97,7 @@ def create_text2music_ui(
         # Get base output directory from environment variable, defaulting to CWD-relative 'outputs'.
         # This default (./outputs) is suitable for non-Docker local development.
         # For Docker, the ACE_OUTPUT_DIR environment variable should be set (e.g., to /app/outputs).
-        _output_base_dir = os.environ.get("ACE_OUTPUT_DIR", "./outputs")
+        output_file_dir = os.environ.get("ACE_OUTPUT_DIR", "./outputs")
-        output_file_dir = os.path.join(_output_base_dir, "text2music")
         if not os.path.isdir(output_file_dir):
             os.makedirs(output_file_dir, exist_ok=True)
         json_files = [f for f in os.listdir(output_file_dir) if f.endswith('.json')]
@@ -235,11 +234,11 @@ def create_text2music_ui(
 
             with gr.Accordion("Advanced Settings", open=False):
                 scheduler_type = gr.Radio(
-                    ["euler", "heun"],
+                    ["euler", "heun", "pingpong"],
-                    value="euler",
+                    value="pingpong",
                     label="Scheduler Type",
                     elem_id="scheduler_type",
-                    info="Scheduler type for the generation. euler is recommended. heun will take more time.",
+                    info="Scheduler type for the generation. pingpong is recommended. heun will take more time.",
                 )
                 cfg_type = gr.Radio(
                     ["cfg", "apg", "cfg_star"],
@@ -911,7 +910,8 @@ def create_text2music_ui(
         )
 
         def load_data(json_file):
-            json_file = os.path.join(output_file_dir, json_file)
+            if isinstance(output_file_dir, str):
+                json_file = os.path.join(output_file_dir, json_file)
             json_data = load_data_func(json_file)
             return json2output(json_data)