Skip to content

refactor: remove url validation #1333

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jupyterjazz merged 2 commits into from
Apr 3, 2023
Merged

refactor: remove url validation #1333

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6cfc37d
refactor: remove url validation
jupyterjazz Apr 3, 2023
5fd1318
refactor: modify docstrings
jupyterjazz Apr 3, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 2 additions & 28 deletions docarray/typing/url/audio_url.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,48 +1,22 @@
import warnings
from typing import TYPE_CHECKING, Any, Optional, Tuple, Type, TypeVar, Union
from typing import Optional, Tuple, TypeVar

from docarray.typing import AudioNdArray
from docarray.typing.bytes.audio_bytes import AudioBytes
from docarray.typing.proto_register import _register_proto
from docarray.typing.url.any_url import AnyUrl
from docarray.typing.url.filetypes import AUDIO_FILE_FORMATS
from docarray.utils._internal.misc import is_notebook

if TYPE_CHECKING:
from pydantic import BaseConfig
from pydantic.fields import ModelField

T = TypeVar('T', bound='AudioUrl')


@_register_proto(proto_type_name='audio_url')
class AudioUrl(AnyUrl):
"""
URL to a audio file.
URL to an audio file.
Can be remote (web) URL, or a local file path.
"""

@classmethod
def validate(
cls: Type[T],
value: Union[T, str, Any],
field: 'ModelField',
config: 'BaseConfig',
) -> T:
import os
from urllib.parse import urlparse

url = super().validate(value, field, config) # basic url validation
path = urlparse(url).path
ext = os.path.splitext(path)[1][1:].lower()

# pass test if extension is valid or no extension
has_audio_extension = ext in AUDIO_FILE_FORMATS or ext == ''

if not has_audio_extension:
raise ValueError('Audio URL must have a valid extension')
return cls(str(url), scheme=None)

def load(self: T) -> Tuple[AudioNdArray, int]:
"""
Load the data from the url into an AudioNdArray and the frame rate.
Expand Down
119 changes: 0 additions & 119 deletions docarray/typing/url/filetypes.py
View file
Open in desktop

This file was deleted.

25 changes: 2 additions & 23 deletions docarray/typing/url/image_url.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import warnings
from typing import TYPE_CHECKING, Any, Optional, Tuple, Type, TypeVar, Union
from typing import TYPE_CHECKING, Optional, Tuple, TypeVar

from docarray.typing import ImageBytes
from docarray.typing.proto_register import _register_proto
Expand All @@ -9,38 +9,17 @@

if TYPE_CHECKING:
from PIL import Image as PILImage
from pydantic import BaseConfig
from pydantic.fields import ModelField


T = TypeVar('T', bound='ImageUrl')

IMAGE_FILE_FORMATS = ('png', 'jpeg', 'jpg')


@_register_proto(proto_type_name='image_url')
class ImageUrl(AnyUrl):
"""
URL to a .png, .jpeg, or .jpg file.
URL to an image file.
Can be remote (web) URL, or a local file path.
Comment thread
jupyterjazz marked this conversation as resolved.
"""

@classmethod
def validate(
cls: Type[T],
value: Union[T, str, Any],
field: 'ModelField',
config: 'BaseConfig',
) -> T:
url = super().validate(value, field, config) # basic url validation
has_image_extension = any(url.endswith(ext) for ext in IMAGE_FILE_FORMATS)
if not has_image_extension:
raise ValueError(
f'Image URL must have one of the following extensions:'
f'{IMAGE_FILE_FORMATS}'
)
return cls(str(url), scheme=None)

def load_pil(self, timeout: Optional[float] = None) -> 'PILImage.Image':
"""
Load the image from the bytes into a `PIL.Image.Image` instance
Expand Down
28 changes: 1 addition & 27 deletions docarray/typing/url/text_url.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,7 @@
from typing import TYPE_CHECKING, Any, Optional, Type, TypeVar, Union
from typing import Optional, TypeVar

from docarray.typing.proto_register import _register_proto
from docarray.typing.url.any_url import AnyUrl
from docarray.typing.url.filetypes import TEXT_FILE_FORMATS

if TYPE_CHECKING:
from pydantic import BaseConfig
from pydantic.fields import ModelField

T = TypeVar('T', bound='TextUrl')

Expand All @@ -18,27 +13,6 @@ class TextUrl(AnyUrl):
Can be remote (web) URL, or a local file path.
"""

@classmethod
def validate(
cls: Type[T],
value: Union[T, str, Any],
field: 'ModelField',
config: 'BaseConfig',
):
import os
from urllib.parse import urlparse

url = super().validate(value, field, config) # basic url validation
path = urlparse(url).path
ext = os.path.splitext(path)[1][1:].lower()

# pass test if extension is valid or no extension
has_valid_text_extension = ext in TEXT_FILE_FORMATS or ext == ''

if not has_valid_text_extension:
raise ValueError('Text URL must have a valid extension')
return cls(str(url), scheme=None)

def load(self, charset: str = 'utf-8', timeout: Optional[float] = None) -> str:
"""
Load the text file into a string.
Expand Down
2 changes: 1 addition & 1 deletion docarray/typing/url/url_3d/mesh_url.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
@_register_proto(proto_type_name='mesh_url')
class Mesh3DUrl(Url3D):
"""
URL to a .obj, .glb, or .ply file containing 3D mesh information.
URL to a file containing 3D mesh information.
Can be remote (web) URL, or a local file path.
"""

Expand Down
2 changes: 1 addition & 1 deletion docarray/typing/url/url_3d/point_cloud_url.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
@_register_proto(proto_type_name='point_cloud_url')
class PointCloud3DUrl(Url3D):
"""
URL to a .obj, .glb, or .ply file containing point cloud information.
URL to a file containing point cloud information.
Can be remote (web) URL, or a local file path.
"""

Expand Down
24 changes: 2 additions & 22 deletions docarray/typing/url/url_3d/url_3d.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,43 +1,23 @@
from abc import ABC
from typing import TYPE_CHECKING, Any, Dict, Optional, Type, TypeVar, Union
from typing import TYPE_CHECKING, Any, Dict, Optional, TypeVar, Union

from docarray.typing.proto_register import _register_proto
from docarray.typing.url.any_url import AnyUrl
from docarray.utils._internal.misc import import_library

if TYPE_CHECKING:
import trimesh
from pydantic import BaseConfig
from pydantic.fields import ModelField

MESH_FILE_FORMATS = ('obj', 'glb', 'ply')

T = TypeVar('T', bound='Url3D')


@_register_proto(proto_type_name='url3d')
class Url3D(AnyUrl, ABC):
"""
URL to a .obj, .glb, or .ply file containing 3D mesh or point cloud information.
URL to a file containing 3D mesh or point cloud information.
Can be remote (web) URL, or a local file path.
Comment thread
jupyterjazz marked this conversation as resolved.
"""

@classmethod
def validate(
cls: Type[T],
value: Union[T, str, Any],
field: 'ModelField',
config: 'BaseConfig',
) -> T:
url = super().validate(value, field, config)
has_mesh_extension = any(url.endswith(ext) for ext in MESH_FILE_FORMATS)
if not has_mesh_extension:
raise ValueError(
f'{cls.__name__} must have one of the following extensions:'
f'{MESH_FILE_FORMATS}'
)
return cls(str(url), scheme=None)

def _load_trimesh_instance(
self: T,
force: Optional[str] = None,
Expand Down
26 changes: 2 additions & 24 deletions docarray/typing/url/video_url.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,43 +1,21 @@
import warnings
from typing import TYPE_CHECKING, Any, Optional, Type, TypeVar, Union
from typing import Optional, TypeVar

from docarray.typing.bytes.video_bytes import VideoBytes, VideoLoadResult
from docarray.typing.proto_register import _register_proto
from docarray.typing.url.any_url import AnyUrl
from docarray.utils._internal.misc import is_notebook

if TYPE_CHECKING:
from pydantic import BaseConfig
from pydantic.fields import ModelField

T = TypeVar('T', bound='VideoUrl')

VIDEO_FILE_FORMATS = ['mp4']


@_register_proto(proto_type_name='video_url')
class VideoUrl(AnyUrl):
"""
URL to a .wav file.
URL to a video file.
Can be remote (web) URL, or a local file path.
Comment thread
jupyterjazz marked this conversation as resolved.
"""

@classmethod
def validate(
cls: Type[T],
value: Union[T, str, Any],
field: 'ModelField',
config: 'BaseConfig',
) -> T:
url = super().validate(value, field, config)
has_video_extension = any(ext in url for ext in VIDEO_FILE_FORMATS)
if not has_video_extension:
raise ValueError(
f'Video URL must have one of the following extensions:'
f'{VIDEO_FILE_FORMATS}'
)
return cls(str(url), scheme=None)

def load(self: T, **kwargs) -> VideoLoadResult:
"""
Load the data from the url into a named Tuple of VideoNdArray, AudioNdArray and
Expand Down
Loading