casacore
TSMCube.h
Go to the documentation of this file.
1 //# TSMCube.h: Tiled hypercube in a table
2 //# Copyright (C) 1995,1996,1997,1999,2000,2001,2002
3 //# Associated Universities, Inc. Washington DC, USA.
4 //#
5 //# This library is free software; you can redistribute it and/or modify it
6 //# under the terms of the GNU Library General Public License as published by
7 //# the Free Software Foundation; either version 2 of the License, or (at your
8 //# option) any later version.
9 //#
10 //# This library is distributed in the hope that it will be useful, but WITHOUT
11 //# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 //# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public
13 //# License for more details.
14 //#
15 //# You should have received a copy of the GNU Library General Public License
16 //# along with this library; if not, write to the Free Software Foundation,
17 //# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
18 //#
19 //# Correspondence concerning AIPS++ should be addressed as follows:
20 //# Internet email: aips2-request@nrao.edu.
21 //# Postal address: AIPS++ Project Office
22 //# National Radio Astronomy Observatory
23 //# 520 Edgemont Road
24 //# Charlottesville, VA 22903-2475 USA
25 //#
26 //# $Id: TSMCube.h 21521 2014-12-10 08:06:42Z gervandiepen $
27 
28 #ifndef TABLES_TSMCUBE_H
29 #define TABLES_TSMCUBE_H
30 
31 
32 //# Includes
33 #include <casacore/casa/aips.h>
34 #include <casacore/tables/DataMan/TSMShape.h>
35 #include <casacore/casa/Containers/Record.h>
36 #include <casacore/casa/Arrays/IPosition.h>
37 #include <casacore/casa/OS/Conversion.h>
38 #include <casacore/casa/iosfwd.h>
39 
40 namespace casacore { //# NAMESPACE CASACORE - BEGIN
41 
42 //# Forward declarations
43 class TiledStMan;
44 class TSMFile;
45 class TSMColumn;
46 class BucketCache;
47 template<class T> class Block;
48 
49 // <summary>
50 // Tiled hypercube in a table
51 // </summary>
52 
53 // <use visibility=local>
54 
55 // <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests="">
56 // </reviewed>
57 
58 // <prerequisite>
59 //# Classes you should understand before using this one.
60 // <li> <linkto class=TiledStMan>TiledStMan</linkto>
61 // <li> <linkto class=ROTiledStManAccessor>ROTiledStManAccessor</linkto>
62 // for a discussion of the maximum cache size
63 // <li> <linkto class=TSMFile>TSMFile</linkto>
64 // <li> <linkto class=BucketCache>BucketCache</linkto>
65 // </prerequisite>
66 
67 // <etymology>
68 // TSMCube represents a hypercube in the Tiled Storage Manager.
69 // </etymology>
70 
71 // <synopsis>
72 // TSMCube defines a tiled hypercube. The data is stored in a TSMFile
73 // object and accessed using a BucketCache object. The hypercube can
74 // be extensible in its last dimension to support tables with a size
75 // which is not known in advance.
76 // <br>
77 // Normally hypercubes share the same TSMFile object, but extensible
78 // hypercubes have their own TSMFile object (to be extensible).
79 // If the hypercolumn has multiple data columns, their cells share the same
80 // tiles. Per tile data column A appears first, thereafter B, etc..
81 // <br>
82 // The data in the cache is held in external format and is converted
83 // when accessed. The alternative would be to hold it in the cache in
84 // local format and convert it when read/written from the file. It was
85 // felt that the latter approach would generate more needless conversions.
86 // <p>
87 // The possible id and coordinate values are stored in a Record
88 // object. They are written in the main hypercube AipsIO file.
89 // <p>
90 // TSMCube uses the maximum cache size set for a Tiled Storage manager.
91 // The description of class
92 // <linkto class=ROTiledStManAccessor>ROTiledStManAccessor</linkto>
93 // contains a discussion about the effect of setting the maximum cache size.
94 // </synopsis>
95 
96 // <motivation>
97 // TSMCube encapsulates all operations on a hypercube.
98 // </motivation>
99 
100 //# <todo asof="$DATE:$">
101 //# A List of bugs, limitations, extensions or planned refinements.
102 //# </todo>
103 
104 
105 class TSMCube
106 {
107 public:
108  // Define the possible access types for TSMDataColumn.
109  enum AccessType {
115  };
116 
117  // Construct the hypercube using the given file with the given shape.
118  // The record contains the id and possible coordinate values.
119  // <br>If the cubeshape is empty, the hypercube is still undefined and
120  // can be added later with setShape. That is only used by TiledCellStMan.
121  // <br> The fileOffset argument is meant for class TiledFileAccess.
122  TSMCube (TiledStMan* stman, TSMFile* file,
123  const IPosition& cubeShape,
124  const IPosition& tileShape,
125  const Record& values,
126  Int64 fileOffset,
127  Bool useDerived = False);
128 
129  // Reconstruct the hypercube by reading its data from the AipsIO stream.
130  // It will link itself to the correct TSMFile. The TSMFile objects
131  // must have been reconstructed in advance.
132  TSMCube (TiledStMan* stman, AipsIO& ios,
133  Bool useDerived = False);
134 
135  virtual ~TSMCube();
136 
137  // Flush the data in the cache.
138  virtual void flushCache();
139 
140  // Clear the cache, so data will be reread.
141  // If wanted, the data is flushed before the cache is cleared.
142  void clearCache (Bool doFlush = True);
143 
144  // Empty the cache.
145  // It will flush the cache as needed and remove all buckets from it
146  // resulting in a possibly large drop in memory used.
147  // It'll also clear the <src>userSetCache_p</src> flag.
148  void emptyCache();
149 
150  // Show the cache statistics.
151  virtual void showCacheStatistics (ostream& os) const;
152 
153  // Put the data of the object into the AipsIO stream.
154  void putObject (AipsIO& ios);
155 
156  // Get the data of the object from the AipsIO stream.
157  // It returns the data manager sequence number, which is -1 if
158  // no file is attached to the cube (for cells without a value).
159  Int getObject (AipsIO& ios);
160 
161  // Resync the object with the data file.
162  // It reads the object, and adjusts the cache.
163  virtual void resync (AipsIO& ios);
164 
165  // Is the hypercube extensible?
166  Bool isExtensible() const;
167 
168  // Get the bucket size (which is the length of a tile in external format).
169  uInt bucketSize() const;
170 
171  // Get the length of a tile in local format.
172  uInt localTileLength() const;
173 
174  // Set the hypercube shape.
175  // This is only possible if the shape was not defined yet.
176  virtual void setShape (const IPosition& cubeShape,
177  const IPosition& tileShape);
178 
179  // Get the shape of the hypercube.
180  const IPosition& cubeShape() const;
181 
182  // Get the shape of the tiles.
183  const IPosition& tileShape() const;
184 
185  // Get the shape of the data cells in the cube.
186  IPosition cellShape() const;
187 
188  // Get the size of a coordinate (i.e. the number of values in it).
189  // If not defined, it returns zero.
190  uInt coordinateSize (const String& coordinateName) const;
191 
192  // Get the record containing the id and coordinate values.
193  // It is used by TSMIdColumn and TSMCoordColumn.
194  // <group>
195  const Record& valueRecord() const;
197  // </group>
198 
199  // Test if the id values match.
200  Bool matches (const PtrBlock<TSMColumn*>& idColSet,
201  const Record& idValues);
202 
203  // Extend the last dimension of the cube with the given number.
204  // The record can contain the coordinates of the elements added.
205  virtual void extend (uInt nr, const Record& coordValues,
206  const TSMColumn* lastCoordColumn);
207 
208  // Extend the coordinates vector for the given coordinate
209  // to the given length with the given coordValues.
210  // It will be initialized to zero if no coordValues are given.
211  // If the coordinate vector does not exist yet, it will be created.
212  void extendCoordinates (const Record& coordValues,
213  const String& coordName, uInt length);
214 
215  // Read or write a section in the cube.
216  // It is assumed that the section buffer is long enough.
217  virtual void accessSection (const IPosition& start, const IPosition& end,
218  char* section, uInt colnr,
219  uInt localPixelSize, uInt externalPixelSize,
220  Bool writeFlag);
221 
222  // Read or write a section in a strided way.
223  // It is assumed that the section buffer is long enough.
224  virtual void accessStrided (const IPosition& start, const IPosition& end,
225  const IPosition& stride,
226  char* section, uInt colnr,
227  uInt localPixelSize, uInt externalPixelSize,
228  Bool writeFlag);
229 
230  // Get the current cache size (in buckets).
231  uInt cacheSize() const;
232 
233  // Calculate the cache size (in buckets) for the given slice
234  // and access path.
235  // <group>
236  uInt calcCacheSize (const IPosition& sliceShape,
237  const IPosition& windowStart,
238  const IPosition& windowLength,
239  const IPosition& axisPath) const;
240  static uInt calcCacheSize (const IPosition& cubeShape,
241  const IPosition& tileShape,
242  Bool extensible,
243  const IPosition& sliceShape,
244  const IPosition& windowStart,
245  const IPosition& windowLength,
246  const IPosition& axisPath,
247  uInt maxCacheSize, uInt bucketSize);
248  // </group>
249 
250  // Set the cache size for the given slice and access path.
251  virtual void setCacheSize (const IPosition& sliceShape,
252  const IPosition& windowStart,
253  const IPosition& windowLength,
254  const IPosition& axisPath,
255  Bool forceSmaller, Bool userSet);
256 
257  // Resize the cache object.
258  // If forceSmaller is False, the cache will only be resized when it grows.
259  // If the given size exceeds the maximum size with more
260  // than 10%, the maximum size will be used.
261  // The cacheSize has to be given in buckets.
262  // <br>The flag <src>userSet</src> inidicates if the cache size is set by
263  // the user (by an Accessor object) or automatically (by TSMDataColumn).
264  virtual void setCacheSize (uInt cacheSize, Bool forceSmaller, Bool userSet);
265 
266  // Validate the cache size (in buckets).
267  // This means it will return the given cache size if smaller
268  // than the maximum cache size. Otherwise the maximum is returned.
269  // <group>
271  static uInt validateCacheSize (uInt cacheSize, uInt maxSize,
272  uInt bucketSize);
273  // </group>
274 
275  // Determine if the user set the cache size (using setCacheSize).
276  Bool userSetCache() const;
277 
278  // Functions for TSMDataColumn to keep track of the last type of
279  // access to a hypercube. It uses it to determine if the cache
280  // has to be reset.
281  // <group>
283  const IPosition& getLastColSlice() const;
284  void setLastColAccess (AccessType type);
285  void setLastColSlice (const IPosition& slice);
286  // </group>
287 
288 protected:
289  // Initialize the various variables.
290  // <group>
291  void setup();
292  void setupNrTiles();
293  // </group>
294 
295  // Adjust the tile shape to the hypercube shape.
296  // A size of 0 gets set to 1.
297  // A tile size > cube size gets set to the cube size.
299  const IPosition& tileShape) const;
300 
301  // Resize the IPosition member variables used in accessSection()
302  // if nrdim_p changes value.
303  void resizeTileSections();
304 
305 private:
306  // Forbid copy constructor.
307  TSMCube (const TSMCube&);
308 
309  // Forbid assignment.
310  TSMCube& operator= (const TSMCube&);
311 
312  // Get the cache object.
313  // This will construct the cache object if not present yet.
315 
316  // Construct the cache object (if not constructed yet).
317  virtual void makeCache();
318 
319  // Resync the cache object.
320  virtual void resyncCache();
321 
322  // Delete the cache object.
323  virtual void deleteCache();
324 
325  // Access a line in a more optimized way.
326  void accessLine (char* section, uInt pixelOffset,
327  uInt localPixelSize,
328  Bool writeFlag, BucketCache* cachePtr,
329  const IPosition& startTile, uInt endTile,
330  const IPosition& startPixelInFirstTile,
331  uInt endPixelInLastTile,
332  uInt lineIndex);
333 
334  // Define the callback functions for the BucketCache.
335  // <group>
336  static char* readCallBack (void* owner, const char* external);
337  static void writeCallBack (void* owner, char* external,
338  const char* local);
339  static char* initCallBack (void* owner);
340  static void deleteCallBack (void* owner, char* buffer);
341  // </group>
342 
343  // Define the functions doing the actual read and write of the
344  // data in the tile and converting it to/from local format.
345  // <group>
346  char* readTile (const char* external);
347  void writeTile (char* external, const char* local);
348  // </group>
349 
350 protected:
351  //# Declare member variables.
352 
353  char * cachedTile_p; // optimization to hold one tile chunk
354 
355  // Pointer to the parent storage manager.
357  // Is the class used directly or only by a derived class only?
359  // The values of the possible id and coordinate columns.
361  // Is the hypercube extensible?
363  // Dimensionality of the hypercube.
365  // Number of tiles in the hypercube.
367  // The shape of the hypercube.
369  // The shape of the tiles in the hypercube.
371  // The number of tiles in each hypercube dimension.
373  // Precomputed tileShape information.
375  // Precomputed tilesPerDim information.
377  // Number of tiles in all but last dimension (used when extending).
379  // The tilesize in pixels.
381  // Pointer to the TSMFile object holding the data.
383  // Offset in the TSMFile object where the data of this hypercube starts.
385  // Offset for each data column in a tile (in external format).
387  // Offset for each data column in a tile (in local format).
389  // The bucket size in bytes (is equal to tile size in bytes).
391  // The tile size in bytes in local format.
393  // The bucket cache.
395  // Did the user set the cache size?
397  // Was the last column access to a cell, slice, or column?
399  // The slice shape of the last column access to a slice.
401 
402  // IPosition variables used in accessSection(); declared here
403  // as member variables to avoid significant construction and
404  // desctruction overhead if they are local to accessSection()
405  // #tiles needed for the section
407  // First tile needed
409  // Last tile needed
411  // First pixel in first tile
413  // Last pixel in first tile
415  // Last pixel in last tile
417 };
418 
419 
420 
422 {
423  if (cache_p == 0) {
424  makeCache();
425  }
426  return cache_p;
427 }
428 inline uInt TSMCube::bucketSize() const
429 {
430  return bucketSize_p;
431 }
433 {
434  return localTileLength_p;
435 }
436 inline const IPosition& TSMCube::cubeShape() const
437 {
438  return cubeShape_p;
439 }
440 inline const IPosition& TSMCube::tileShape() const
441 {
442  return tileShape_p;
443 }
444 inline const Record& TSMCube::valueRecord() const
445 {
446  return values_p;
447 }
449 {
450  return values_p;
451 }
453 {
454  return userSetCache_p;
455 }
457 {
458  return lastColAccess_p;
459 }
460 inline const IPosition& TSMCube::getLastColSlice() const
461 {
462  return lastColSlice_p;
463 }
465 {
466  lastColAccess_p = type;
467 }
468 inline void TSMCube::setLastColSlice (const IPosition& slice)
469 {
470  lastColSlice_p.resize (slice.nelements());
471  lastColSlice_p = slice;
472 }
473 
474 
475 
476 
477 } //# NAMESPACE CASACORE - END
478 
479 #endif
static void deleteCallBack(void *owner, char *buffer)
A Vector of integers, for indexing into Array<T> objects.
Definition: IPosition.h:119
Bool extensible_p
Is the hypercube extensible?
Definition: TSMCube.h:362
void setLastColAccess(AccessType type)
Definition: TSMCube.h:464
IPosition lastColSlice_p
The slice shape of the last column access to a slice.
Definition: TSMCube.h:400
long long Int64
Define the extra non-standard types used by Casacore (like proposed uSize, Size)
Definition: aipsxtype.h:38
int Int
Definition: aipstype.h:50
virtual void showCacheStatistics(ostream &os) const
Show the cache statistics.
Cache for buckets in a part of a file.
Definition: BucketCache.h:217
void accessLine(char *section, uInt pixelOffset, uInt localPixelSize, Bool writeFlag, BucketCache *cachePtr, const IPosition &startTile, uInt endTile, const IPosition &startPixelInFirstTile, uInt endPixelInLastTile, uInt lineIndex)
Access a line in a more optimized way.
const Record & valueRecord() const
Get the record containing the id and coordinate values.
Definition: TSMCube.h:444
Bool isExtensible() const
Is the hypercube extensible?
Bool useDerived_p
Is the class used directly or only by a derived class only?
Definition: TSMCube.h:358
IPosition cellShape() const
Get the shape of the data cells in the cube.
Tiled hypercube in a table.
Definition: TSMCube.h:105
IPosition adjustTileShape(const IPosition &cubeShape, const IPosition &tileShape) const
Adjust the tile shape to the hypercube shape.
TSMCube(TiledStMan *stman, TSMFile *file, const IPosition &cubeShape, const IPosition &tileShape, const Record &values, Int64 fileOffset, Bool useDerived=False)
Construct the hypercube using the given file with the given shape.
AipsIO is the object persistency mechanism of Casacore.
Definition: AipsIO.h:168
TiledStMan * stmanPtr_p
Pointer to the parent storage manager.
Definition: TSMCube.h:356
IPosition endPixelInLastTile_p
Last pixel in last tile.
Definition: TSMCube.h:416
AccessType
Define the possible access types for TSMDataColumn.
Definition: TSMCube.h:109
Base class for Tiled Storage Manager classes.
Definition: TiledStMan.h:107
IPosition cubeShape_p
The shape of the hypercube.
Definition: TSMCube.h:368
uInt localTileLength_p
The tile size in bytes in local format.
Definition: TSMCube.h:392
uInt bucketSize() const
Get the bucket size (which is the length of a tile in external format).
Definition: TSMCube.h:428
static char * initCallBack(void *owner)
uInt localTileLength() const
Get the length of a tile in local format.
Definition: TSMCube.h:432
IPosition tileShape_p
The shape of the tiles in the hypercube.
Definition: TSMCube.h:370
AccessType getLastColAccess() const
Functions for TSMDataColumn to keep track of the last type of access to a hypercube.
Definition: TSMCube.h:456
TSMShape expandedTilesPerDim_p
Precomputed tilesPerDim information.
Definition: TSMCube.h:376
Block< uInt > localOffset_p
Offset for each data column in a tile (in local format).
Definition: TSMCube.h:388
void extendCoordinates(const Record &coordValues, const String &coordName, uInt length)
Extend the coordinates vector for the given coordinate to the given length with the given coordValues...
Bool userSetCache() const
Determine if the user set the cache size (using setCacheSize).
Definition: TSMCube.h:452
IPosition nrTileSection_p
IPosition variables used in accessSection(); declared here as member variables to avoid significant c...
Definition: TSMCube.h:406
void setup()
Initialize the various variables.
IPosition tilesPerDim_p
The number of tiles in each hypercube dimension.
Definition: TSMCube.h:372
IPosition startPixelInFirstTile_p
First pixel in first tile.
Definition: TSMCube.h:412
virtual void accessSection(const IPosition &start, const IPosition &end, char *section, uInt colnr, uInt localPixelSize, uInt externalPixelSize, Bool writeFlag)
Read or write a section in the cube.
uInt nelements() const
The number of elements in this IPosition.
Definition: IPosition.h:532
uInt nrdim_p
Dimensionality of the hypercube.
Definition: TSMCube.h:364
uInt nrTilesSubCube_p
Number of tiles in all but last dimension (used when extending).
Definition: TSMCube.h:378
uInt cacheSize() const
Get the current cache size (in buckets).
Int64 fileOffset_p
Offset in the TSMFile object where the data of this hypercube starts.
Definition: TSMCube.h:384
Int getObject(AipsIO &ios)
Get the data of the object from the AipsIO stream.
virtual void resync(AipsIO &ios)
Resync the object with the data file.
virtual void extend(uInt nr, const Record &coordValues, const TSMColumn *lastCoordColumn)
Extend the last dimension of the cube with the given number.
virtual void flushCache()
Flush the data in the cache.
AccessType lastColAccess_p
Was the last column access to a cell, slice, or column?
Definition: TSMCube.h:398
BucketCache * getCache()
Get the cache object.
Definition: TSMCube.h:421
Bool userSetCache_p
Did the user set the cache size?
Definition: TSMCube.h:396
LatticeExprNode length(const LatticeExprNode &expr, const LatticeExprNode &axis)
2-argument function to get the length of an axis.
void resizeTileSections()
Resize the IPosition member variables used in accessSection() if nrdim_p changes value.
void clearCache(Bool doFlush=True)
Clear the cache, so data will be reread.
uInt calcCacheSize(const IPosition &sliceShape, const IPosition &windowStart, const IPosition &windowLength, const IPosition &axisPath) const
Calculate the cache size (in buckets) for the given slice and access path.
A hierarchical collection of named fields of various types.
Definition: Record.h:180
bool Bool
Define the standard types used by Casacore.
Definition: aipstype.h:42
const IPosition & tileShape() const
Get the shape of the tiles.
Definition: TSMCube.h:440
IPosition startTile_p
First tile needed.
Definition: TSMCube.h:408
virtual ~TSMCube()
A column in the Tiled Storage Manager.
Definition: TSMColumn.h:97
TSMShape expandedTileShape_p
Precomputed tileShape information.
Definition: TSMCube.h:374
File object for Tiled Storage Manager.
Definition: TSMFile.h:81
void putObject(AipsIO &ios)
Put the data of the object into the AipsIO stream.
uInt validateCacheSize(uInt cacheSize) const
Validate the cache size (in buckets).
const Bool False
Definition: aipstype.h:44
uInt coordinateSize(const String &coordinateName) const
Get the size of a coordinate (i.e.
A drop-in replacement for Block<T*>.
Definition: Block.h:861
virtual void resyncCache()
Resync the cache object.
const IPosition & getLastColSlice() const
Definition: TSMCube.h:460
char * cachedTile_p
Definition: TSMCube.h:353
TSMCube & operator=(const TSMCube &)
Forbid assignment.
uInt nrTiles_p
Number of tiles in the hypercube.
Definition: TSMCube.h:366
IPosition endPixelInFirstTile_p
Last pixel in first tile.
Definition: TSMCube.h:414
uInt tileSize_p
The tilesize in pixels.
Definition: TSMCube.h:380
Expanded IPosition for shapes.
Definition: TSMShape.h:77
BucketCache * cache_p
The bucket cache.
Definition: TSMCube.h:394
virtual void makeCache()
Construct the cache object (if not constructed yet).
virtual void accessStrided(const IPosition &start, const IPosition &end, const IPosition &stride, char *section, uInt colnr, uInt localPixelSize, uInt externalPixelSize, Bool writeFlag)
Read or write a section in a strided way.
TSMFile * filePtr_p
Pointer to the TSMFile object holding the data.
Definition: TSMCube.h:382
const IPosition & cubeShape() const
Get the shape of the hypercube.
Definition: TSMCube.h:436
virtual void setCacheSize(const IPosition &sliceShape, const IPosition &windowStart, const IPosition &windowLength, const IPosition &axisPath, Bool forceSmaller, Bool userSet)
Set the cache size for the given slice and access path.
void setLastColSlice(const IPosition &slice)
Definition: TSMCube.h:468
virtual void deleteCache()
Delete the cache object.
String: the storage and methods of handling collections of characters.
Definition: String.h:223
char * readTile(const char *external)
Define the functions doing the actual read and write of the data in the tile and converting it to/fro...
IPosition endTile_p
Last tile needed.
Definition: TSMCube.h:410
void resize(uInt newSize, Bool copy=True)
Old values are copied on resize if copy==True.
virtual void setShape(const IPosition &cubeShape, const IPosition &tileShape)
Set the hypercube shape.
static char * readCallBack(void *owner, const char *external)
Define the callback functions for the BucketCache.
static void writeCallBack(void *owner, char *external, const char *local)
Bool matches(const PtrBlock< TSMColumn *> &idColSet, const Record &idValues)
Test if the id values match.
Record values_p
The values of the possible id and coordinate columns.
Definition: TSMCube.h:360
void emptyCache()
Empty the cache.
Record & rwValueRecord()
Definition: TSMCube.h:448
uInt bucketSize_p
The bucket size in bytes (is equal to tile size in bytes).
Definition: TSMCube.h:390
const Bool True
Definition: aipstype.h:43
this file contains all the compiler specific defines
Definition: mainpage.dox:28
Block< uInt > externalOffset_p
Offset for each data column in a tile (in external format).
Definition: TSMCube.h:386
void writeTile(char *external, const char *local)
unsigned int uInt
Definition: aipstype.h:51