Coverage for /home/ubuntu/hidebound/python/hidebound/exporters/girder_exporter.py: 100%
57 statements
« prev ^ index » next coverage.py v7.5.4, created at 2024-07-05 23:50 +0000
1from typing import Any, Dict, List, Union # noqa F401
3from pathlib import Path
5from schematics import Model
6from schematics.types import IntType, ListType, StringType, URLType
7import girder_client
9from hidebound.exporters.exporter_base import ExporterBase
10import hidebound.core.validators as vd
11# ------------------------------------------------------------------------------
14class GirderConfig(Model):
15 '''
16 A class for validating configurations supplied to GirderExporter.
18 Attributes:
19 name (str): Name of exporter. Must be 'girder'.
20 api_key (str): Girder API key.
21 root_id (str): ID of folder or collection under which all data will
22 be exported.
23 root_type (str, optional): Root entity type. Default: collection.
24 Options: folder, collection
25 host (str, optional): Docker host URL address. Default: http://0.0.0.0
26 port (int, optional): Docker host port. Default: 8180.
27 metadata_types (list, optional): List of metadata types for export.
28 Default: [asset, file].
29 '''
30 name = StringType(
31 required=True, validators=[lambda x: vd.is_eq(x, 'girder')]
32 ) # type: StringType
33 api_key = StringType(required=True) # type: StringType
34 root_id = StringType(required=True) # type: StringType
35 root_type = StringType(
36 required=True,
37 default='collection',
38 validators=[lambda x: vd.is_in([x], ['collection', 'folder'])]
39 ) # type: StringType
40 host = URLType(required=True, default='http://0.0.0.0') # type: URLType
41 port = IntType(
42 required=True,
43 default=8180,
44 validators=[
45 lambda x: vd.is_lt(x, 65536),
46 lambda x: vd.is_gt(x, 1023),
47 ]
48 ) # type: IntType
49 metadata_types = ListType(
50 StringType(validators=[vd.is_metadata_type]),
51 required=True,
52 default=['asset', 'file']
53 )
56class GirderExporter(ExporterBase):
57 '''
58 Export for Girder asset framework.
59 '''
60 @staticmethod
61 def from_config(config, client=None):
62 # type: (Dict, Any) -> GirderExporter
63 '''
64 Construct a GirderExporter from a given config.
66 Args:
67 config (dict): Config dictionary.
68 client (object, optional): Client instance, for testing.
69 Default: None.
71 Raises:
72 DataError: If config is invalid.
74 Returns:
75 GirderExporter: GirderExporter instance.
76 '''
77 return GirderExporter(client=client, **config)
79 def __init__(
80 self,
81 api_key,
82 root_id,
83 root_type='collection',
84 host='http://0.0.0.0',
85 port=8180,
86 client=None,
87 metadata_types=['asset', 'file'],
88 **kwargs,
89 ):
90 # type: (str, str, str, str, int, Any, List[str], Any) -> None
91 '''
92 Constructs a GirderExporter instances and creates a Girder client.
94 Args:
95 api_key (str): Girder API key.
96 root_id (str): ID of folder or collection under which all data will
97 be exported.
98 root_type (str, optional): Root entity type. Default: collection.
99 Options: folder, collection
100 host (str, optional): Docker host URL address.
101 Default: http://0.0.0.0.
102 port (int, optional): Docker host port. Default: 8180.
103 client (object, optional): Client instance, for testing.
104 Default: None.
105 metadata_types (list[str], optional): Metadata types to export.
106 Default: [asset, file].
107 ):
109 Raises:
110 DataError: If config is invalid.
111 '''
112 # sudo ip addr show docker0 | grep inet | grep docker0 | awk '{print $2}' | sed 's/\/.*//'
113 # will give you the ip address of the docker network which binds to
114 # localhost
116 metadata_types = list(
117 filter(lambda x: x in ['asset', 'file'], metadata_types)
118 )
119 super().__init__(metadata_types=metadata_types)
121 config = dict(
122 name='girder',
123 api_key=api_key,
124 root_id=root_id,
125 root_type=root_type,
126 host=host,
127 port=port,
128 metadata_types=metadata_types,
129 )
130 GirderConfig(config).validate()
132 self._url = f'{host}:{port}/api/v1' # type: str
134 if client is None:
135 client = girder_client.GirderClient(apiUrl=self._url) # pragma: no cover
136 client.authenticate(apiKey=api_key) # pragma: no cover
137 self._client = client # type: Any
139 self._root_id = root_id # type: str
140 self._root_type = root_type # type: str
142 def _export_dirs(self, dirpath, metadata={}, exists_ok=False):
143 # type: (Union[str, Path], Dict, bool) -> Dict
144 '''
145 Recursively export all the directories found in given path.
147 Args:
148 dirpath (Path or str): Directory paht to be exported.
149 metadata (dict, optional): Metadata to be appended to final
150 directory. Default: {}.
152 Returns:
153 dict: Response (contains _id key).
154 '''
155 dirs = Path(dirpath).parts # type: Any
156 dirs = list(filter(lambda x: x != '/', dirs))
158 # if dirpath has no parents then export to root with metadata
159 if len(dirs) == 1:
160 return self._client.createFolder(
161 self._root_id,
162 dirs[0],
163 metadata=metadata,
164 reuseExisting=exists_ok,
165 parentType=self._root_type,
166 )
168 # if dirpath has parents then export all parent directories
169 response = dict(_id=self._root_id)
170 parent_type = self._root_type
171 for dir_ in dirs[:-1]:
172 response = self._client.createFolder(
173 response['_id'],
174 dir_,
175 reuseExisting=True,
176 parentType=parent_type
177 )
178 parent_type = 'folder'
180 # then export last directory with metadata
181 return self._client.createFolder(
182 response['_id'],
183 dirs[-1],
184 metadata=metadata,
185 reuseExisting=exists_ok,
186 parentType='folder',
187 )
189 def _export_content(self, metadata):
190 # type: (dict) -> Any
191 '''
192 Export file content and metadata to Girder.
193 Metadata must contain these fields:
194 * filepath_relative
195 * filename
196 * filepath
198 Args:
199 metadata (dict): File metadata.
201 Returns:
202 object: Response.
203 '''
204 filepath = metadata['filepath_relative']
205 filename = metadata['filename']
206 parent_dir = Path(filepath).parent
207 response = self._export_dirs(parent_dir, exists_ok=True)
209 # folder error will always be raised before duplicate file conflict is
210 # encountered, so don't test for duplicate files within directory
212 meta = metadata
213 if 'file' not in self._metadata_types:
214 meta = {}
216 response = self._client.createItem(
217 response['_id'],
218 filename,
219 metadata=meta,
220 reuseExisting=True,
221 )
222 response = self._client\
223 .uploadFileToItem(response['_id'], metadata['filepath'])
224 return response
226 def _export_asset(self, metadata):
227 # type: (dict) -> None
228 '''
229 Export asset metadata to Girder.
230 Metadata must contain these fields:
231 * asset_type
232 * asset_path_relative
234 Args:
235 metadata (dict): Asset metadata.
236 '''
237 if metadata['asset_type'] != 'file':
238 self._export_dirs(
239 metadata['asset_path_relative'],
240 metadata=metadata,
241 exists_ok=True,
242 )
244 def _export_file(self, metadata):
245 # type: (dict) -> None
246 '''
247 Exports content from file metadata in hidebound/metadata/file.
249 Args:
250 metadata (dict): File metadata.
251 '''
252 pass # pragma: no cover
254 def _export_asset_chunk(self, metadata):
255 # type: (List[dict]) -> None
256 '''
257 Exports content from asset log in hidebound/metadata/asset-chunk.
259 Args:
260 metadata (list[dict]): Asset metadata chunk.
261 '''
262 pass # pragma: no cover
264 def _export_file_chunk(self, metadata):
265 # type: (List[dict]) -> None
266 '''
267 Exports content from file log in hidebound/metadata/file-chunk.
269 Args:
270 metadata (list[dict]): File metadata chunk.
271 '''
272 pass # pragma: no cover