1 ///
2 module dpq2.conv.from_d_types;
3 
4 @safe:
5 
6 public import dpq2.conv.arrays : isArrayType, toValue, isStaticArrayString;
7 public import dpq2.conv.geometric : isGeometricType, toValue;
8 import dpq2.conv.time : POSTGRES_EPOCH_DATE, TimeStamp, TimeStampUTC, TimeOfDayWithTZ, Interval;
9 import dpq2.oids : detectOidTypeFromNative, oidConvTo, OidType;
10 import dpq2.value : Value, ValueFormat;
11 
12 import std.bitmanip: nativeToBigEndian, BitArray, append;
13 import std.datetime.date: Date, DateTime, TimeOfDay;
14 import std.datetime.systime: SysTime;
15 import std.datetime.timezone: LocalTime, TimeZone, UTC;
16 import std.traits: isImplicitlyConvertible, isNumeric, isInstanceOf, OriginalType, Unqual, isSomeString;
17 import std.typecons : Nullable;
18 import std.uuid: UUID;
19 import vibe.data.json: Json;
20 import money: currency;
21 
22 /// Converts Nullable!T to Value
23 Value toValue(T)(T v)
24 if (is(T == Nullable!R, R) && !(isArrayType!(typeof(v.get))) && !isGeometricType!(typeof(v.get)))
25 {
26     if (v.isNull)
27         return Value(ValueFormat.BINARY, detectOidTypeFromNative!T);
28     else
29         return toValue(v.get);
30 }
31 
32 /// ditto
33 Value toValue(T)(T v)
34 if (is(T == Nullable!R, R) && (isArrayType!(typeof(v.get))))
35 {
36     import dpq2.conv.arrays : arrToValue = toValue; // deprecation import workaround
37     import std.range : ElementType;
38 
39     if (v.isNull)
40         return Value(ValueFormat.BINARY, detectOidTypeFromNative!(ElementType!(typeof(v.get))).oidConvTo!"array");
41     else
42         return arrToValue(v.get);
43 }
44 
45 /// ditto
46 Value toValue(T)(T v)
47 if (is(T == Nullable!R, R) && isGeometricType!(typeof(v.get)))
48 {
49     import dpq2.conv.geometric : geoToValue = toValue; // deprecation import workaround
50 
51     if (v.isNull)
52         return Value(ValueFormat.BINARY, detectOidTypeFromNative!T);
53     else
54         return geoToValue(v.get);
55 }
56 
57 ///
58 Value toValue(T)(T v)
59 if(isNumeric!(T))
60 {
61     return Value(v.nativeToBigEndian.dup, detectOidTypeFromNative!T, false, ValueFormat.BINARY);
62 }
63 
64 /// Convert money.currency to PG value
65 ///
66 /// Caution: here is no check of fractional precision while conversion!
67 /// See also: PostgreSQL's "lc_monetary" description and "money" package description
68 Value toValue(T)(T v)
69 if(isInstanceOf!(currency, T) &&  T.amount.sizeof == 8)
70 {
71     return Value(v.amount.nativeToBigEndian.dup, OidType.Money, false, ValueFormat.BINARY);
72 }
73 
74 unittest
75 {
76     import dpq2.conv.to_d_types: PGTestMoney;
77 
78     const pgtm = PGTestMoney(-123.45);
79 
80     Value v = pgtm.toValue;
81 
82     assert(v.oidType == OidType.Money);
83     assert(v.as!PGTestMoney == pgtm);
84 }
85 
86 /// Convert std.bitmanip.BitArray to PG value
87 Value toValue(T)(T v) @trusted
88 if(is(Unqual!T == BitArray))
89 {
90     import std.array : appender;
91     import core.bitop : bitswap;
92 
93     size_t len = v.length / 8 + (v.length % 8 ? 1 : 0);
94     auto data = cast(size_t[])v;
95     auto buffer = appender!(const ubyte[])();
96     buffer.append!uint(cast(uint)v.length);
97     foreach (d; data[0 .. v.dim])
98     {
99         // DMD Issue 19693
100         version(DigitalMars)
101             auto ntb = nativeToBigEndian(softBitswap(d));
102         else
103             auto ntb = nativeToBigEndian(bitswap(d));
104         foreach (b; ntb[0 .. len])
105         {
106             buffer.append!ubyte(b);
107         }
108 
109     }
110     return Value(buffer.data.dup, detectOidTypeFromNative!T, false, ValueFormat.BINARY);
111 }
112 
113 /// Reverses the order of bits - needed because of dmd Issue 19693
114 /// https://issues.dlang.org/show_bug.cgi?id=19693
115 package N softBitswap(N)(N x) pure
116     if (is(N == uint) || is(N == ulong))
117 {
118     import core.bitop : bswap;
119     // swap 1-bit pairs:
120     enum mask1 = cast(N) 0x5555_5555_5555_5555L;
121     x = ((x >> 1) & mask1) | ((x & mask1) << 1);
122     // swap 2-bit pairs:
123     enum mask2 = cast(N) 0x3333_3333_3333_3333L;
124     x = ((x >> 2) & mask2) | ((x & mask2) << 2);
125     // swap 4-bit pairs:
126     enum mask4 = cast(N) 0x0F0F_0F0F_0F0F_0F0FL;
127     x = ((x >> 4) & mask4) | ((x & mask4) << 4);
128 
129     // reverse the order of all bytes:
130     x = bswap(x);
131 
132     return x;
133 }
134 
135 @trusted unittest
136 {
137     import std.bitmanip : BitArray;
138 
139     auto varbit = BitArray([1,0,1,1,0]);
140 
141     Value v = varbit.toValue;
142 
143     assert(v.oidType == OidType.VariableBitString);
144     assert(v.as!BitArray == varbit);
145 
146     // test softBitswap
147     assert (softBitswap!uint( 0x8000_0100 ) == 0x0080_0001);
148     foreach (i; 0 .. 32)
149         assert (softBitswap!uint(1 << i) == 1 << 32 - i - 1);
150 
151     assert (softBitswap!ulong( 0b1000000000000000000000010000000000000000100000000000000000000001)
152             == 0b1000000000000000000000010000000000000000100000000000000000000001);
153     assert (softBitswap!ulong( 0b1110000000000000000000010000000000000000100000000000000000000001)
154         == 0b1000000000000000000000010000000000000000100000000000000000000111);
155     foreach (i; 0 .. 64)
156         assert (softBitswap!ulong(1UL << i) == 1UL << 64 - i - 1);
157 
158 }
159 
160 /**
161     Converts types implicitly convertible to string to PG Value.
162     Note that if string is null it is written as an empty string.
163     If NULL is a desired DB value, Nullable!string can be used instead.
164 */
165 Value toValue(T)(T v, ValueFormat valueFormat = ValueFormat.BINARY) @trusted
166 if(isSomeString!T || isStaticArrayString!T)
167 {
168     static if(is(T == string))
169     {
170         import std.string : representation;
171 
172         static assert(isImplicitlyConvertible!(T, string));
173         auto buf = (cast(string) v).representation;
174 
175         if(valueFormat == ValueFormat.TEXT) buf ~= 0; // for prepareArgs only
176 
177         return Value(buf, OidType.Text, false, valueFormat);
178     }
179     else
180     {
181         // convert to a string
182         import std.conv : to;
183         return toValue(v.to!string, valueFormat);
184     }
185 }
186 
187 /// Constructs Value from array of bytes
188 Value toValue(T)(T v)
189 if(is(T : immutable(ubyte)[]))
190 {
191     return Value(v, detectOidTypeFromNative!(ubyte[]), false, ValueFormat.BINARY);
192 }
193 
194 /// Constructs Value from boolean
195 Value toValue(T : bool)(T v) @trusted
196 if (!is(T == Nullable!R, R))
197 {
198     immutable ubyte[] buf = [ v ? 1 : 0 ];
199 
200     return Value(buf, detectOidTypeFromNative!T, false, ValueFormat.BINARY);
201 }
202 
203 /// Constructs Value from Date
204 Value toValue(T)(T v)
205 if (is(Unqual!T == Date))
206 {
207     import std.conv: to;
208     import dpq2.value;
209     import dpq2.conv.time: POSTGRES_EPOCH_JDATE;
210 
211     long mj_day = v.modJulianDay;
212 
213     // max days isn't checked because Phobos Date days value always fits into Postgres Date
214     if (mj_day < -POSTGRES_EPOCH_JDATE)
215         throw new ValueConvException(
216                 ConvExceptionType.DATE_VALUE_OVERFLOW,
217                 "Date value doesn't fit into Postgres binary Date",
218                 __FILE__, __LINE__
219             );
220 
221     enum mj_pg_epoch = POSTGRES_EPOCH_DATE.modJulianDay;
222     long days = mj_day - mj_pg_epoch;
223 
224     return Value(nativeToBigEndian(days.to!int).dup, OidType.Date, false);
225 }
226 
227 private long convTimeOfDayToPG(in TimeOfDay v) pure
228 {
229     return ((60L * v.hour + v.minute) * 60 + v.second) * 1_000_000;
230 }
231 
232 /// Constructs Value from TimeOfDay
233 Value toValue(T)(T v)
234 if (is(Unqual!T == TimeOfDay))
235 {
236     return Value(v.convTimeOfDayToPG.nativeToBigEndian.dup, OidType.Time);
237 }
238 
239 /// Constructs Value from TimeOfDay
240 Value toValue(T)(T v)
241 if (is(Unqual!T == TimeOfDayWithTZ))
242 {
243     const buf = v.time.convTimeOfDayToPG.nativeToBigEndian ~ v.tzSec.nativeToBigEndian;
244     assert(buf.length == 12);
245 
246     return Value(buf.dup, OidType.TimeWithZone);
247 }
248 
249 /// Constructs Value from Interval
250 Value toValue(T)(T v)
251 if (is(Unqual!T == Interval))
252 {
253     const buf = v.usecs.nativeToBigEndian ~ v.days.nativeToBigEndian ~ v.months.nativeToBigEndian;
254     assert(buf.length == 16);
255 
256     return Value(buf.dup, OidType.TimeInterval);
257 }
258 
259 /// Constructs Value from TimeStamp or from TimeStampUTC
260 Value toValue(T)(T v)
261 if (is(Unqual!T == TimeStamp) || is(Unqual!T == TimeStampUTC))
262 {
263     long us; /// microseconds
264 
265     if(v.isLater) // infinity
266         us = us.max;
267     else if(v.isEarlier) // -infinity
268         us = us.min;
269     else
270     {
271         enum mj_pg_epoch = POSTGRES_EPOCH_DATE.modJulianDay;
272         long j = modJulianDayForIntYear(v.date.year, v.date.month, v.date.day) - mj_pg_epoch;
273         us = (((j * 24 + v.time.hour) * 60 + v.time.minute) * 60 + v.time.second) * 1_000_000 + v.fracSec.total!"usecs";
274     }
275 
276     return Value(
277             nativeToBigEndian(us).dup,
278             is(Unqual!T == TimeStamp) ? OidType.TimeStamp : OidType.TimeStampWithZone,
279             false
280         );
281 }
282 
283 private auto modJulianDayForIntYear(const int year, const ubyte month, const short day) pure
284 {
285     // Wikipedia magic:
286 
287     const a = (14 - month) / 12;
288     const y = year + 4800 - a;
289     const m = month + a * 12 - 3;
290 
291     const jd = day + (m*153+2)/5 + y*365 + y/4 - y/100 + y/400 - 32045;
292 
293     return jd - 2_400_001;
294 }
295 unittest
296 {
297     assert(modJulianDayForIntYear(1858, 11, 17) == 0);
298     assert(modJulianDayForIntYear(2010, 8, 24) == 55_432);
299     assert(modJulianDayForIntYear(1999, 7, 6) == 51_365);
300 }
301 
302 /++
303     Constructs Value from DateTime
304     It uses Timestamp without TZ as a resulting PG type
305 +/
306 Value toValue(T)(T v)
307 if (is(Unqual!T == DateTime))
308 {
309     return TimeStamp(v).toValue;
310 }
311 
312 /++
313     Constructs Value from SysTime
314     Note that SysTime has a precision in hnsecs and PG TimeStamp in usecs.
315     It means that PG value will have 10 times lower precision.
316     And as both types are using long for internal storage it also means that PG TimeStamp can store greater range of values than SysTime.
317 +/
318 Value toValue(T)(T v)
319 if (is(Unqual!T == SysTime))
320 {
321     import dpq2.value: ValueConvException, ConvExceptionType;
322     import core.time;
323     import std.conv: to;
324 
325     long usecs;
326     int hnsecs;
327     v.fracSecs.split!("usecs", "hnsecs")(usecs, hnsecs);
328 
329     if(hnsecs)
330         throw new ValueConvException(
331             ConvExceptionType.TOO_PRECISE,
332             "fracSecs have 1 microsecond resolution but contains "~v.fracSecs.to!string
333         );
334 
335     long us = (v - SysTime(POSTGRES_EPOCH_DATE, UTC())).total!"usecs";
336 
337     return Value(nativeToBigEndian(us).dup, OidType.TimeStampWithZone, false);
338 }
339 
340 /// Constructs Value from UUID
341 Value toValue(T)(T v)
342 if (is(Unqual!T == UUID))
343 {
344     return Value(v.data.dup, OidType.UUID);
345 }
346 
347 /// Constructs Value from Json
348 Value toValue(T)(T v)
349 if (is(Unqual!T == Json))
350 {
351     auto r = toValue(v.toString);
352     r.oidType = OidType.Json;
353 
354     return r;
355 }
356 
357 Value toRecordValue(Value[] elements)
358 {
359     import std.array : appender;
360     auto buffer = appender!(ubyte[])();
361     buffer ~= nativeToBigEndian!int(cast(int)elements.length)[];
362     foreach (element; elements)
363     {
364         buffer ~= nativeToBigEndian!int(element.oidType)[];
365         if (element.isNull) {
366             buffer ~= nativeToBigEndian!int(-1)[];
367         } else {
368             buffer ~= nativeToBigEndian!int(cast(int)element.data.length)[];
369             buffer ~= element.data;
370         }
371     }
372 
373     return Value(buffer.data.idup, OidType.Record);
374 }
375 
376 version(unittest)
377 import dpq2.conv.to_d_types : as, deserializeRecord;
378 
379 unittest
380 {
381     import std.stdio;
382     Value[] vals = [toValue(17.34), toValue(Nullable!long(17)), toValue(Nullable!long.init)];
383     Value v = vals.toRecordValue;
384     assert(deserializeRecord(v) == vals);
385 }
386 
387 unittest
388 {
389     Value v = toValue(cast(short) 123);
390 
391     assert(v.oidType == OidType.Int2);
392     assert(v.as!short == 123);
393 }
394 
395 unittest
396 {
397     Value v = toValue(-123.456);
398 
399     assert(v.oidType == OidType.Float8);
400     assert(v.as!double == -123.456);
401 }
402 
403 unittest
404 {
405     Value v = toValue("Test string");
406 
407     assert(v.oidType == OidType.Text);
408     assert(v.as!string == "Test string");
409 }
410 
411 // string Null values
412 @system unittest
413 {
414     {
415         import core.exception: AssertError;
416         import std.exception: assertThrown;
417 
418         auto v = Nullable!string.init.toValue;
419         assert(v.oidType == OidType.Text);
420         assert(v.isNull);
421 
422         assertThrown!AssertError(v.as!string);
423         assert(v.as!(Nullable!string).isNull);
424     }
425 
426     {
427         string s;
428         auto v = s.toValue;
429         assert(v.oidType == OidType.Text);
430         assert(!v.isNull);
431     }
432 }
433 
434 unittest
435 {
436     immutable ubyte[] buf = [0, 1, 2, 3, 4, 5];
437     Value v = toValue(buf);
438 
439     assert(v.oidType == OidType.ByteArray);
440     assert(v.as!(const ubyte[]) == buf);
441 }
442 
443 unittest
444 {
445     Value t = toValue(true);
446     Value f = toValue(false);
447 
448     assert(t.as!bool == true);
449     assert(f.as!bool == false);
450 }
451 
452 unittest
453 {
454     Value v = toValue(Nullable!long(1));
455     Value nv = toValue(Nullable!bool.init);
456 
457     assert(!v.isNull);
458     assert(v.oidType == OidType.Int8);
459     assert(v.as!long == 1);
460 
461     assert(nv.isNull);
462     assert(nv.oidType == OidType.Bool);
463 }
464 
465 unittest
466 {
467     import std.datetime : DateTime;
468 
469     Value v = toValue(Nullable!TimeStamp(TimeStamp(DateTime(2017, 1, 2))));
470 
471     assert(!v.isNull);
472     assert(v.oidType == OidType.TimeStamp);
473 }
474 
475 unittest
476 {
477     // Date: '2018-1-15'
478     auto d = Date(2018, 1, 15);
479     auto v = toValue(d);
480 
481     assert(v.oidType == OidType.Date);
482     assert(v.as!Date == d);
483 }
484 
485 unittest
486 {
487     auto d = immutable Date(2018, 1, 15);
488     auto v = toValue(d);
489 
490     assert(v.oidType == OidType.Date);
491     assert(v.as!Date == d);
492 }
493 
494 unittest
495 {
496     // Date: '2000-1-1'
497     auto d = Date(2000, 1, 1);
498     auto v = toValue(d);
499 
500     assert(v.oidType == OidType.Date);
501     assert(v.as!Date == d);
502 }
503 
504 unittest
505 {
506     // Date: '0010-2-20'
507     auto d = Date(10, 2, 20);
508     auto v = toValue(d);
509 
510     assert(v.oidType == OidType.Date);
511     assert(v.as!Date == d);
512 }
513 
514 unittest
515 {
516     // Date: max (always fits into Postgres Date)
517     auto d = Date.max;
518     auto v = toValue(d);
519 
520     assert(v.oidType == OidType.Date);
521     assert(v.as!Date == d);
522 }
523 
524 unittest
525 {
526     // Date: min (overflow)
527     import std.exception: assertThrown;
528     import dpq2.value: ValueConvException;
529 
530     auto d = Date.min;
531     assertThrown!ValueConvException(d.toValue);
532 }
533 
534 unittest
535 {
536     // DateTime
537     auto d = const DateTime(2018, 2, 20, 1, 2, 3);
538     auto v = toValue(d);
539 
540     assert(v.oidType == OidType.TimeStamp);
541     assert(v.as!DateTime == d);
542 }
543 
544 unittest
545 {
546     // Nullable!DateTime
547     import std.typecons : nullable;
548     auto d = nullable(DateTime(2018, 2, 20, 1, 2, 3));
549     auto v = toValue(d);
550 
551     assert(v.oidType == OidType.TimeStamp);
552     assert(v.as!(Nullable!DateTime) == d);
553 
554     d.nullify();
555     v = toValue(d);
556     assert(v.oidType == OidType.TimeStamp);
557     assert(v.as!(Nullable!DateTime).isNull);
558 }
559 
560 unittest
561 {
562     // TimeOfDay: '14:29:17'
563     auto tod = TimeOfDay(14, 29, 17);
564     auto v = toValue(tod);
565 
566     assert(v.oidType == OidType.Time);
567     assert(v.as!TimeOfDay == tod);
568 }
569 
570 unittest
571 {
572     auto t = TimeOfDayWithTZ(
573         TimeOfDay(14, 29, 17),
574         -3600 * 7 // Negative means TZ == +07
575     );
576 
577     auto v = toValue(t);
578 
579     assert(v.oidType == OidType.TimeWithZone);
580     assert(v.as!TimeOfDayWithTZ == t);
581 }
582 
583 unittest
584 {
585     auto t = Interval(
586         -123,
587         -456,
588         -789
589     );
590 
591     auto v = toValue(t);
592 
593     assert(v.oidType == OidType.TimeInterval);
594     assert(v.as!Interval == t);
595 }
596 
597 unittest
598 {
599     // SysTime: '2017-11-13T14:29:17.075678Z'
600     auto t = SysTime.fromISOExtString("2017-11-13T14:29:17.075678Z");
601     auto v = toValue(t);
602 
603     assert(v.oidType == OidType.TimeStampWithZone);
604     assert(v.as!SysTime == t);
605 }
606 
607 unittest
608 {
609     import core.time: dur;
610     import std.exception: assertThrown;
611     import dpq2.value: ValueConvException;
612 
613     auto t = SysTime.fromISOExtString("2017-11-13T14:29:17.075678Z");
614     t += dur!"hnsecs"(1);
615 
616     // TOO_PRECISE
617     assertThrown!ValueConvException(t.toValue);
618 }
619 
620 unittest
621 {
622     import core.time : usecs;
623     import std.datetime.date : DateTime;
624 
625     // TimeStamp: '2017-11-13 14:29:17.075678'
626     auto t = TimeStamp(DateTime(2017, 11, 13, 14, 29, 17), 75_678.usecs);
627     auto v = toValue(t);
628 
629     assert(v.oidType == OidType.TimeStamp);
630     assert(v.as!TimeStamp == t);
631 }
632 
633 unittest
634 {
635     auto j = Json(["foo":Json("bar")]);
636     auto v = j.toValue;
637 
638     assert(v.oidType == OidType.Json);
639     assert(v.as!Json == j);
640 
641     auto nj = Nullable!Json(j);
642     auto nv = nj.toValue;
643     assert(nv.oidType == OidType.Json);
644     assert(!nv.as!(Nullable!Json).isNull);
645     assert(nv.as!(Nullable!Json).get == j);
646 }
647 
648 unittest
649 {
650     import dpq2.conv.to_d_types : as;
651     char[2] arr;
652     auto v = arr.toValue();
653     assert(v.oidType == OidType.Text);
654     assert(!v.isNull);
655 
656     auto varr = v.as!string;
657     assert(varr.length == 2);
658 }