feat(binary): лимит памяти для ДвоичныеДанные из конфигурации (#1667)

src/OneScript.StandardLibrary/Binary/BinaryDataConstants.cs

@@ -0,0 +1,25 @@
+/*----------------------------------------------------------
+This Source Code Form is subject to the terms of the
+Mozilla Public License, v.2.0. If a copy of the MPL
+was not distributed with this file, You can obtain one
+at http://mozilla.org/MPL/2.0/.
+----------------------------------------------------------*/
+
+using System;
+
+namespace OneScript.StandardLibrary.Binary
+{
+    public static class BinaryDataConstants
+    {
+        /// <summary>
+        /// Максимальный размер массива, доступный в среде выполнения.
+        /// Де-факто он чуть меньше 2Гб, он же Int32.MaxValue, поэтому используется системная константа <see cref="Array.MaxLength"/>
+        /// </summary>
+        public static readonly int SYSTEM_IN_MEMORY_LIMIT = Array.MaxLength;
+
+        /// <summary>
+        /// Размер двоичных данных, хранимый в памяти по умолчанию.
+        /// </summary>
+        public const int DEFAULT_IN_MEMORY_LIMIT = 1024 * 1024 * 50;
+    }
+}

src/OneScript.StandardLibrary/Binary/BinaryDataContext.cs

@@ -24,32 +24,32 @@ public sealed class BinaryDataContext : AutoContext<BinaryDataContext>, IDisposa
         private byte[] _buffer;
         private BackingTemporaryFile _backingFile;
 
-        public BinaryDataContext(string filename)
+        public BinaryDataContext(string filename, int inMemLimit)
         {
             using var fs = new FileStream(filename, FileMode.Open, FileAccess.Read);
-            ReadFromStream(fs);
+            ReadFromStream(fs, inMemLimit);
         }
 
         public BinaryDataContext(byte[] buffer)
         {
             _buffer = buffer ?? throw new ArgumentNullException(nameof(buffer));
         }
 
-        public BinaryDataContext(Stream stream)
+        public BinaryDataContext(Stream stream, int inMemLimit)
         {
             long pos = 0;
-            ReadFromStream(stream);
+            ReadFromStream(stream, inMemLimit);
             stream.Position = pos;
         }
-
+        
         /// <summary>
         /// Признак хранения данных в памяти
         /// </summary>
         public bool InMemory => _backingFile == null;
 
-        private void ReadFromStream(Stream stream)
+        private void ReadFromStream(Stream stream, int inMemLimit)
         {
-            if (stream.Length < FileBackingConstants.DEFAULT_MEMORY_LIMIT)
+            if (stream.Length < inMemLimit)
             {
                 LoadToBuffer(stream);
             }
@@ -197,9 +197,9 @@ public GenericStream OpenStreamForRead()
         }
 
         [ScriptConstructor(Name = "На основании файла")]
-        public static BinaryDataContext Constructor(string filename)
+        public static BinaryDataContext Constructor(TypeActivationContext ctx, string filename)
         {
-            return new BinaryDataContext(filename);
+            return new BinaryDataContext(filename, ctx.Services.Resolve<IBinaryDataMemoryLimit>().MaxBytesInMemory);
         }
 
         public override bool Equals(BslValue other)

src/OneScript.StandardLibrary/Binary/BinaryDataOptions.cs

@@ -0,0 +1,101 @@
+// /*----------------------------------------------------------
+// This Source Code Form is subject to the terms of the
+// Mozilla Public License, v.2.0. If a copy of the MPL
+// was not distributed with this file, You can obtain one
+// at http://mozilla.org/MPL/2.0/.
+// ----------------------------------------------------------*/
+
+using System.Globalization;
+using ScriptEngine;
+using ScriptEngine.Hosting;
+
+namespace OneScript.StandardLibrary.Binary
+{
+    /// <summary>
+    /// Инкапсулирует логику чтения и хранения настроек двоичных данных
+    /// </summary>
+    public class BinaryDataOptions : IBinaryDataMemoryLimit
+    {
+        public const string IN_MEMORY_LIMIT_KEY_NAME = "binaryData.inMemoryMaxSize";
+        public const string IN_MEMORY_MAX_MAGIC = "max";
+
+        public BinaryDataOptions(KeyValueConfig config)
+        {
+            var configValue = config[IN_MEMORY_LIMIT_KEY_NAME];
+            MaxBytesInMemory = ResolveFromConfigString(configValue);
+        }
+
+        public int MaxBytesInMemory { get; }
+
+        private static int ResolveFromConfigString(string rawValue)
+        {
+            if (string.IsNullOrWhiteSpace(rawValue))
+                return BinaryDataConstants.DEFAULT_IN_MEMORY_LIMIT;
+
+            if (rawValue.Trim() == IN_MEMORY_MAX_MAGIC)
+                return BinaryDataConstants.SYSTEM_IN_MEMORY_LIMIT;
+
+            if (!TryParseByteSize(rawValue.Trim(), out var bytes))
+            {
+                SystemLogger.Write($"Invalid value for {IN_MEMORY_LIMIT_KEY_NAME}: {rawValue}");
+                return BinaryDataConstants.DEFAULT_IN_MEMORY_LIMIT;
+            }
+
+            if (bytes <= 0 || bytes >= BinaryDataConstants.SYSTEM_IN_MEMORY_LIMIT)
+            {
+                SystemLogger.Write($"Value for {IN_MEMORY_LIMIT_KEY_NAME} must be between 1 and {BinaryDataConstants.SYSTEM_IN_MEMORY_LIMIT - 1}: {bytes}");
+                return BinaryDataConstants.DEFAULT_IN_MEMORY_LIMIT;
+            }
+
+            return (int)bytes;
+        }
+
+        private static bool TryParseByteSize(string value, out long bytes)
+        {
+            bytes = 0;
+
+            if (value.Length == 0)
+                return false;
+
+            var suffix = value[value.Length - 1];
+            long multiplier = 1;
+            var numberPart = value;
+
+            const long KILOBYTES = 1024L;
+            const long MEGABYTES = KILOBYTES * 1024L;
+            const long GIGABYTES = MEGABYTES * 1024L;
+
+            switch (suffix)
+            {
+                case 'k':
+                case 'K':
+                    multiplier = KILOBYTES;
+                    numberPart = value.Substring(0, value.Length - 1).TrimEnd();
+                    break;
+                case 'm':
+                case 'M':
+                    multiplier = MEGABYTES;
+                    numberPart = value.Substring(0, value.Length - 1).TrimEnd();
+                    break;
+                case 'g':
+                case 'G':
+                    multiplier = GIGABYTES;
+                    numberPart = value.Substring(0, value.Length - 1).TrimEnd();
+                    break;
+            }
+
+            if (numberPart.Length == 0)
+                return false;
+
+            if (!long.TryParse(numberPart, NumberStyles.AllowLeadingWhite | NumberStyles.AllowTrailingWhite, CultureInfo.InvariantCulture, out var number))
+                return false;
+
+            if (number <= 0)
+                return false;
+
+            bytes = number * multiplier;
+
+            return true;
+        }
+    }
+}

src/OneScript.StandardLibrary/Binary/FileBackingStream.cs

@@ -20,13 +20,9 @@ public sealed class FileBackingStream : Stream
 
         private string _backingFileName;
 
-        public FileBackingStream() : this(FileBackingConstants.DEFAULT_MEMORY_LIMIT)
-        {
-        }
-
         public FileBackingStream(int inMemoryLimit, int capacity = 0)
         {
-            if (inMemoryLimit == FileBackingConstants.SYSTEM_IN_MEMORY_LIMIT)
+            if (inMemoryLimit == BinaryDataConstants.SYSTEM_IN_MEMORY_LIMIT)
                 throw new ArgumentException("Use MemoryStream instead");
 
             _inMemoryLimit = inMemoryLimit;
@@ -85,6 +81,7 @@ public override long Position
         public bool HasBackingFile => _backingFileName != null;
 
         public string FileName => _backingFileName;
+        public int InMemoryThreshold => _inMemoryLimit;
 
         public void SwitchToMemory()
         {

src/OneScript.StandardLibrary/Binary/GlobalBinaryData.cs

@@ -11,6 +11,7 @@ This Source Code Form is subject to the terms of the
 using System.Text;
 using OneScript.Contexts;
 using OneScript.Exceptions;
+using OneScript.Types;
 using ScriptEngine.Machine;
 using ScriptEngine.Machine.Contexts;
 namespace OneScript.StandardLibrary.Binary
@@ -21,6 +22,13 @@ namespace OneScript.StandardLibrary.Binary
     [GlobalContext(Category = "Процедуры и функции работы с двоичными данными")]
     public sealed class GlobalBinaryData : GlobalContextBase<GlobalBinaryData>
     {
+        private readonly int _memoryLimitMaxBytesInMemory;
+
+        private GlobalBinaryData(int memoryLimitMaxBytesInMemory)
+        {
+            _memoryLimitMaxBytesInMemory = memoryLimitMaxBytesInMemory;
+        }
+
         private static byte[] HexStringToByteArray(string hex)
         {
             var newHex = System.Text.RegularExpressions.Regex.Replace(hex, @"[^0-9A-Fa-f]", "");
@@ -142,9 +150,9 @@ private static void CheckAndThrowIfNull<T>(AutoContext<T> obj, int argNumber, st
                 throw RuntimeException.InvalidArgumentType(argNumber, argName);
         }
 
-        public static IAttachableContext CreateInstance()
+        public static IAttachableContext CreateInstance(IBinaryDataMemoryLimit memoryLimit)
         {
-            return new GlobalBinaryData();
+            return new GlobalBinaryData(memoryLimit.MaxBytesInMemory);
         }
 
         /// <summary>
@@ -174,7 +182,7 @@ public BinaryDataContext ConcatBinaryData(ArrayImpl array)
             }
             stream.Position = 0;
 
-            return new BinaryDataContext(stream);
+            return new BinaryDataContext(stream, _memoryLimitMaxBytesInMemory);
         }
 
         /// <summary>
@@ -244,7 +252,7 @@ public BinaryDataContext GetBinaryDataFromString(string str, IValue encoding = n
             stream.Write(inputString, 0, inputString.Length);
             stream.Position = 0;
 
-            return new BinaryDataContext(stream);
+            return new BinaryDataContext(stream, _memoryLimitMaxBytesInMemory);
         }
 
         /// <summary>

src/OneScript.StandardLibrary/Binary/IBinaryDataMemoryLimit.cs

@@ -0,0 +1,18 @@
+/*----------------------------------------------------------
+This Source Code Form is subject to the terms of the
+Mozilla Public License, v.2.0. If a copy of the MPL
+was not distributed with this file, You can obtain one
+at http://mozilla.org/MPL/2.0/.
+----------------------------------------------------------*/
+
+namespace OneScript.StandardLibrary.Binary
+{
+    /// <summary>
+    /// Лимит объёма данных в памяти для объектов «ДвоичныеДанные» и смежных потоков
+    /// до выгрузки во временный файл (байты).
+    /// </summary>
+    public interface IBinaryDataMemoryLimit
+    {
+        int MaxBytesInMemory { get; }
+    }
+}

src/OneScript.StandardLibrary/EngineBuilderExtensions.cs

@@ -5,6 +5,7 @@ This Source Code Form is subject to the terms of the
 at http://mozilla.org/MPL/2.0/.
 ----------------------------------------------------------*/
 
+using OneScript.StandardLibrary.Binary;
 using OneScript.StandardLibrary.Collections;
 using ScriptEngine.Hosting;
 using ScriptEngine.Machine;
@@ -13,9 +14,15 @@ namespace OneScript.StandardLibrary
 {
     public static class EngineBuilderExtensions
     {
+        public static IEngineBuilder UseBinaryDataOptions(this IEngineBuilder builder)
+        {
+            builder.Services.RegisterSingleton<IBinaryDataMemoryLimit, BinaryDataOptions>();
+            return builder;
+        }
+
         public static ExecutionContext AddStandardLibrary(this ExecutionContext env)
         {
             return env.AddAssembly(typeof(ArrayImpl).Assembly);
         }
     }
-}
+}

src/OneScript.StandardLibrary/Http/HttpRequestBodyBinary.cs

@@ -17,42 +17,19 @@ namespace OneScript.StandardLibrary.Http
 {
     class HttpRequestBodyBinary : IHttpRequestBody
     {
-        private readonly FileBackingStream _storage = new FileBackingStream();
+        private readonly FileBackingStream _storage;
 
-        public HttpRequestBodyBinary()
+        internal HttpRequestBodyBinary(int inMemoryBodyLimit)
         {
+            _storage = new FileBackingStream(inMemoryBodyLimit);
         }
 
-        public HttpRequestBodyBinary(BinaryDataContext data)
+        internal HttpRequestBodyBinary(int inMemoryBodyLimit, BinaryDataContext data)
         {
+            _storage = new FileBackingStream(inMemoryBodyLimit);
             data.CopyTo(_storage);
         }
 
-        public HttpRequestBodyBinary(string body, IValue encoding = null,
-            ByteOrderMarkUsageEnum bomUsage = ByteOrderMarkUsageEnum.Auto)
-        {
-            var useBom = bomUsage == ByteOrderMarkUsageEnum.Auto ||
-                         bomUsage == ByteOrderMarkUsageEnum.Use;
-
-            Encoding encoder;
-            if (encoding == null)
-            {
-                encoder = new UTF8Encoding(useBom);
-            }
-            else if (encoding.SystemType == BasicTypes.String)
-            {
-                var utfs = new List<string> {"utf-16", "utf-32"};
-                encoder = TextEncodingEnum.GetEncoding(encoding, utfs.Contains(encoding.ToString()) && useBom);
-            }
-            else
-            {
-                encoder = TextEncodingEnum.GetEncoding(encoding);
-            }
-
-            var byteArray = encoder.GetBytes(body);
-            _storage.Write(byteArray, 0, byteArray.Length);
-        }
-
         public IValue GetAsString()
         {
             _storage.Seek(0, SeekOrigin.Begin);
@@ -63,7 +40,7 @@ public IValue GetAsString()
         public IValue GetAsBinary()
         {
             _storage.Seek(0, SeekOrigin.Begin);
-            return new BinaryDataContext(_storage);
+            return new BinaryDataContext(_storage, _storage.InMemoryThreshold);
         }
 
         public IValue GetAsFilename()