Skip to content

Register now for

# Reference for `ultralytics/nn/modules/utils.py`

Note

This file is available at https://github.com/ultralytics/ultralytics/blob/main/ultralytics/nn/modules/utils.py. If you spot a problem please help fix it by contributing a Pull Request 🛠️. Thank you 🙏!

## `` ultralytics.nn.modules.utils._get_clones

    _get_clones(module, n)

Create a list of cloned modules from the given module.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`module` | `Module` | The module to be cloned. | required
`n` | `int` | Number of clones to create. | required

Returns:

Type | Description
--- | ---
`ModuleList` | A ModuleList containing n clones of the input module.

Examples:

    >>> import torch.nn as nn
    >>> layer = nn.Linear(10, 10)
    >>> clones = _get_clones(layer, 3)
    >>> len(clones)
    3

Source code in `ultralytics/nn/modules/utils.py`

    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    def _get_clones(module, n):
        """
        Create a list of cloned modules from the given module.

        Args:
            module (nn.Module): The module to be cloned.
            n (int): Number of clones to create.

        Returns:
            (nn.ModuleList): A ModuleList containing n clones of the input module.

        Examples:
            >>> import torch.nn as nn
            >>> layer = nn.Linear(10, 10)
            >>> clones = _get_clones(layer, 3)
            >>> len(clones)
            3
        """
        return nn.ModuleList([copy.deepcopy(module) for _ in range(n)])

## `` ultralytics.nn.modules.utils.bias_init_with_prob

    bias_init_with_prob(prior_prob=0.01)

Initialize conv/fc bias value according to a given probability value.

This function calculates the bias initialization value based on a prior probability using the inverse error function. It's commonly used in object detection models to initialize classification layers with a specific positive prediction probability.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`prior_prob` | `float` | Prior probability for bias initialization. | `0.01`

Returns:

Type | Description
--- | ---
`float` | Bias initialization value calculated from the prior probability.

Examples:

    >>> bias = bias_init_with_prob(0.01)
    >>> print(f"Bias initialization value: {bias:.4f}")
    Bias initialization value: -4.5951

Source code in `ultralytics/nn/modules/utils.py`

    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    46
    47
    48
    49
    50
    51
    52
    53
    54
    55
    def bias_init_with_prob(prior_prob=0.01):
        """
        Initialize conv/fc bias value according to a given probability value.

        This function calculates the bias initialization value based on a prior probability using the inverse error function.
        It's commonly used in object detection models to initialize classification layers with a specific positive prediction
        probability.

        Args:
            prior_prob (float, optional): Prior probability for bias initialization.

        Returns:
            (float): Bias initialization value calculated from the prior probability.

        Examples:
            >>> bias = bias_init_with_prob(0.01)
            >>> print(f"Bias initialization value: {bias:.4f}")
            Bias initialization value: -4.5951
        """
        return float(-np.log((1 - prior_prob) / prior_prob))  # return bias_init

## `` ultralytics.nn.modules.utils.linear_init

    linear_init(module)

Initialize the weights and biases of a linear module.

This function initializes the weights of a linear module using a uniform distribution within bounds calculated from the input dimension. If the module has a bias, it is also initialized.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`module` | `Module` | Linear module to initialize. | required

Returns:

Type | Description
--- | ---
`Module` | The initialized module.

Examples:

    >>> import torch.nn as nn
    >>> linear = nn.Linear(10, 5)
    >>> initialized_linear = linear_init(linear)

Source code in `ultralytics/nn/modules/utils.py`

    58
    59
    60
    61
    62
    63
    64
    65
    66
    67
    68
    69
    70
    71
    72
    73
    74
    75
    76
    77
    78
    79
    def linear_init(module):
        """
        Initialize the weights and biases of a linear module.

        This function initializes the weights of a linear module using a uniform distribution within bounds calculated
        from the input dimension. If the module has a bias, it is also initialized.

        Args:
            module (nn.Module): Linear module to initialize.

        Returns:
            (nn.Module): The initialized module.

        Examples:
            >>> import torch.nn as nn
            >>> linear = nn.Linear(10, 5)
            >>> initialized_linear = linear_init(linear)
        """
        bound = 1 / math.sqrt(module.weight.shape[0])
        uniform_(module.weight, -bound, bound)
        if hasattr(module, "bias") and module.bias is not None:
            uniform_(module.bias, -bound, bound)

## `` ultralytics.nn.modules.utils.inverse_sigmoid

    inverse_sigmoid(x, eps=1e-05)

Calculate the inverse sigmoid function for a tensor.

This function applies the inverse of the sigmoid function to a tensor, which is useful in various neural network operations, particularly in attention mechanisms and coordinate transformations.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`x` | `Tensor` | Input tensor with values in range [0, 1]. | required
`eps` | `float` | Small epsilon value to prevent numerical instability. | `1e-05`

Returns:

Type | Description
--- | ---
`Tensor` | Tensor after applying the inverse sigmoid function.

Examples:

    >>> x = torch.tensor([0.2, 0.5, 0.8])
    >>> inverse_sigmoid(x)
    tensor([-1.3863,  0.0000,  1.3863])

Source code in `ultralytics/nn/modules/utils.py`

     82
     83
     84
     85
     86
     87
     88
     89
     90
     91
     92
     93
     94
     95
     96
     97
     98
     99
    100
    101
    102
    103
    104
    def inverse_sigmoid(x, eps=1e-5):
        """
        Calculate the inverse sigmoid function for a tensor.

        This function applies the inverse of the sigmoid function to a tensor, which is useful in various neural network
        operations, particularly in attention mechanisms and coordinate transformations.

        Args:
            x (torch.Tensor): Input tensor with values in range [0, 1].
            eps (float, optional): Small epsilon value to prevent numerical instability.

        Returns:
            (torch.Tensor): Tensor after applying the inverse sigmoid function.

        Examples:
            >>> x = torch.tensor([0.2, 0.5, 0.8])
            >>> inverse_sigmoid(x)
            tensor([-1.3863,  0.0000,  1.3863])
        """
        x = x.clamp(min=0, max=1)
        x1 = x.clamp(min=eps)
        x2 = (1 - x).clamp(min=eps)
        return torch.log(x1 / x2)

## `` ultralytics.nn.modules.utils.multi_scale_deformable_attn_pytorch

    multi_scale_deformable_attn_pytorch(
        value: Tensor,
        value_spatial_shapes: Tensor,
        sampling_locations: Tensor,
        attention_weights: Tensor,
    ) -> torch.Tensor

Implement multi-scale deformable attention in PyTorch.

This function performs deformable attention across multiple feature map scales, allowing the model to attend to different spatial locations with learned offsets.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`value` | `Tensor` | The value tensor with shape (bs, num_keys, num_heads, embed_dims). | required
`value_spatial_shapes` | `Tensor` | Spatial shapes of the value tensor with shape (num_levels, 2). | required
`sampling_locations` | `Tensor` | The sampling locations with shape (bs, num_queries, num_heads, num_levels, num_points, 2). | required
`attention_weights` | `Tensor` | The attention weights with shape (bs, num_queries, num_heads, num_levels, num_points). | required

Returns:

Type | Description
--- | ---
`Tensor` | The output tensor with shape (bs, num_queries, embed_dims).

References

https://github.com/IDEA-Research/detrex/blob/main/detrex/layers/multi_scale_deform_attn.py

Source code in `ultralytics/nn/modules/utils.py`

    107
    108
    109
    110
    111
    112
    113
    114
    115
    116
    117
    118
    119
    120
    121
    122
    123
    124
    125
    126
    127
    128
    129
    130
    131
    132
    133
    134
    135
    136
    137
    138
    139
    140
    141
    142
    143
    144
    145
    146
    147
    148
    149
    150
    151
    152
    153
    154
    155
    156
    157
    158
    159
    160
    161
    162
    163
    164
    def multi_scale_deformable_attn_pytorch(
        value: torch.Tensor,
        value_spatial_shapes: torch.Tensor,
        sampling_locations: torch.Tensor,
        attention_weights: torch.Tensor,
    ) -> torch.Tensor:
        """
        Implement multi-scale deformable attention in PyTorch.

        This function performs deformable attention across multiple feature map scales, allowing the model to attend to
        different spatial locations with learned offsets.

        Args:
            value (torch.Tensor): The value tensor with shape (bs, num_keys, num_heads, embed_dims).
            value_spatial_shapes (torch.Tensor): Spatial shapes of the value tensor with shape (num_levels, 2).
            sampling_locations (torch.Tensor): The sampling locations with shape
                (bs, num_queries, num_heads, num_levels, num_points, 2).
            attention_weights (torch.Tensor): The attention weights with shape
                (bs, num_queries, num_heads, num_levels, num_points).

        Returns:
            (torch.Tensor): The output tensor with shape (bs, num_queries, embed_dims).

        References:
            https://github.com/IDEA-Research/detrex/blob/main/detrex/layers/multi_scale_deform_attn.py
        """
        bs, _, num_heads, embed_dims = value.shape
        _, num_queries, num_heads, num_levels, num_points, _ = sampling_locations.shape
        value_list = value.split([H_ * W_ for H_, W_ in value_spatial_shapes], dim=1)
        sampling_grids = 2 * sampling_locations - 1
        sampling_value_list = []
        for level, (H_, W_) in enumerate(value_spatial_shapes):
            # bs, H_*W_, num_heads, embed_dims ->
            # bs, H_*W_, num_heads*embed_dims ->
            # bs, num_heads*embed_dims, H_*W_ ->
            # bs*num_heads, embed_dims, H_, W_
            value_l_ = value_list[level].flatten(2).transpose(1, 2).reshape(bs * num_heads, embed_dims, H_, W_)
            # bs, num_queries, num_heads, num_points, 2 ->
            # bs, num_heads, num_queries, num_points, 2 ->
            # bs*num_heads, num_queries, num_points, 2
            sampling_grid_l_ = sampling_grids[:, :, :, level].transpose(1, 2).flatten(0, 1)
            # bs*num_heads, embed_dims, num_queries, num_points
            sampling_value_l_ = F.grid_sample(
                value_l_, sampling_grid_l_, mode="bilinear", padding_mode="zeros", align_corners=False
            )
            sampling_value_list.append(sampling_value_l_)
        # (bs, num_queries, num_heads, num_levels, num_points) ->
        # (bs, num_heads, num_queries, num_levels, num_points) ->
        # (bs, num_heads, 1, num_queries, num_levels*num_points)
        attention_weights = attention_weights.transpose(1, 2).reshape(
            bs * num_heads, 1, num_queries, num_levels * num_points
        )
        output = (
            (torch.stack(sampling_value_list, dim=-2).flatten(-2) * attention_weights)
            .sum(-1)
            .view(bs, num_heads * embed_dims, num_queries)
        )
        return output.transpose(1, 2).contiguous()

📅 Created 1 year ago ✏️ Updated 1 year ago