Home Dashboard

This web application is hosted on my local IIS server and is displayed on a Raspberry Pi w/ touchscreen. It’s 100% custom, using Angular Material with a dotnet backend. It communicates with several IoT APIs, including:

  • Philips Hue
  • Samsung SmartThings
  • Hydrawise
  • OpenMeteo
  • Roku

Lego Deals

This application displays the current prices of Lego sets at major retailers. Every hour, a scheduled task scrapes 29 retailer websites for their discounted Lego sets, and those prices are saved to a database. I started collecting the hourly price history in December 2022 and it continues to run today. I’m able to set alerts via Telegram when the price of a particular set drops below a given threshold. The application also keeps track of which sets I already own, which ones I am interested in, and which ones I never want to see. I can switch to a view that only shows sets I care about.

Lego set details, like number of pieces, are pulled from Rebrickable’s daily dataset. Once per day, a scheduled task downloads the latest Rebrickable dataset and adds the new entries to my database. MSRP is scraped from the Lego website and the BestBuy API. BestBuy just happens to have the best API of all the retailers. And it’s FREE!

Another page in the application displays Ebay buy-it-now, free shipping listings. Due to inconsistency with Ebay data, these prices are not tracked in my database. Ebay deals are displayed on a totally separate page from the retailer listings; however, the Ebay page can be filtered by the same criteria.

The backend, dotnet API and Angular application are hosted on a local IIS server. Several scheduled tasks, like MSRP updates, Rebrickable datasets, alerts, etc. run on the same server. The MySQL database is hosted on Bluehost.

Configuration Extension Examples

AutoMapper Configuration

using AutoMapper;
using Christopher.Snay.Sample.Configuration.AutoMapperProfiles;
using Christopher.Snay.Sample.Configuration.AutoMapperProfiles.Dtos;
using Microsoft.Extensions.DependencyInjection;

namespace Christopher.Snay.Sample.Configuration
{
	public static class AutoMapperConfiguration
	{
		public static void AddAutoMapperConfiguration(this IServiceCollection services)
		{
			services.AddSingleton(new MapperConfiguration(config =>
			{
				config.AddProfile<SampleMapperProfile1>();
				config.AddProfile<SampleMapperProfile2>();		
			}).CreateMapper());
		}
	}
}

CORS Configuration

using Microsoft.AspNetCore.Builder;
using Microsoft.Extensions.DependencyInjection;

namespace Christopher.Snay.Sample.Configuration
{
	public static class CorsConfiguration
	{
		public static IApplicationBuilder UseCorsConfiguration(this IApplicationBuilder app)
		{
			return app.UseCors("CorsPolicy");
		}

		public static IServiceCollection AddCorsConfiguration(this IServiceCollection services)
		{
			var origins = new List<string>
			{
				"http://localhost:4200",
				
				"http://sample-server",
			};

			return services.AddCors(options =>
			{
				options.AddPolicy("CorsPolicy", builder =>
				{
					builder
						.WithOrigins(origins.ToArray())
						.AllowAnyMethod()
						.AllowAnyHeader()
						.AllowCredentials();
				});
			});
		}
	}
}

HttpClientFactory Configuration

using System.Net.Http.Headers;
using System.Net.Mime;
using Christopher.Snay.Sample.Services.RetailerServices;
using Microsoft.Extensions.DependencyInjection;

namespace Christopher.Snay.Sample.Configuration
{
	public static class HttpConfiguration
	{
		public static void AddHttpClientConfiguration(this IServiceCollection services)
		{
			services.AddHttpClient(nameof(ISampleService1), x => x.BaseAddress
				= new Uri("https://www.api.sample.com/search"));

			services.AddHttpClient(nameof(ISampleService2), x => x.BaseAddress
				= new Uri("https://www.api.sample2.com/search"));

			services.AddHttpClient(nameof(ISampleService3), x =>
			{
				x.BaseAddress = new Uri("https://www.api.sample3.com/search");
				x.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue(new ProductHeaderValue("Mozilla", "5.0")));
				x.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue(MediaTypeNames.Application.Json));
				x.DefaultRequestHeaders.AcceptEncoding.Add(new StringWithQualityHeaderValue("gzip"));
				x.DefaultRequestHeaders.Host = "www.sample.com";
				x.DefaultRequestHeaders.Connection.Add("keep-alive");
				x.DefaultRequestHeaders.Add("cookie", "Bearer ABC123");
			}
		}
	}
}

Scrape rendered HTML with .NET6 C#

using HtmlAgilityPack;
using OpenQA.Selenium;
using OpenQA.Selenium.Chrome;
using OpenQA.Selenium.PhantomJS;

namespace Christopher.Snay.Sample.Services.Scrapers
{
	internal class ChromeScraper : IChromeScraper
	{
		public HtmlDocument ScrapeHtml(Uri url)
		{
			return ScrapeHtml(url.ToString());
		}

		public HtmlDocument ScrapeHtml(string url)
		{
			HtmlDocument doc = new();

			using (PhantomJSDriverService driverService = PhantomJSDriverService.CreateDefaultService())
			{
				driverService.HideCommandPromptWindow = true;
				driverService.LoadImages = false;
				driverService.IgnoreSslErrors = true;

				driverService.Start();

				using IWebDriver driver = new ChromeDriver();

				driver.Navigate().GoToUrl(url);

				Thread.Sleep(3000);

				doc.LoadHtml(driver.PageSource);
			}

			return doc;
		}
	}

	public interface IChromeScraper
	{
		HtmlDocument ScrapeHtml(string url);
		HtmlDocument ScrapeHtml(Uri url);
	}
}

Requirements – The following executables must be in /bin

  • phantonjs.exe
  • chrome.exe
  • chromedriver.exe
  • + the chrome portable binaries directory, currently named \107.0.5304.88
<!-- To copy directly to bin without being placed in a sub-folder -->
<ItemGroup>
	<ContentWithTargetPath Include="Assets\phantomjs.exe">
		<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
		<TargetPath>phantomjs.exe</TargetPath>
	</ContentWithTargetPath>
	<ContentWithTargetPath Include="Assets\chrome.exe">
		<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
		<TargetPath>chrome.exe</TargetPath>
	</ContentWithTargetPath>
	<ContentWithTargetPath Include="Assets\chromedriver.exe">
		<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
		<TargetPath>chromedriver.exe</TargetPath>
	</ContentWithTargetPath>

	<None Include="Assets\phantomjs.exe" />
	<None Include="Assets\chrome.exe" />
	<None Include="Assets\chromedriver.exe" />
</ItemGroup>

I’ve used Chrome portable to avoid having to install Chrome. If Chrome is installed, the chrome.exe steps can probably be skipped.

A Better Generic Repository for Entity Framework

using System.Linq.Expressions;
using Christopher.Snay.Sample.DataAccess.Database;
using Microsoft.EntityFrameworkCore;

namespace Christopher.Snay.Sample.DataAccess.Repositories
{
	internal sealed class Repository<TEntity> : IRepository<TEntity> where TEntity : class
	{
		private readonly SampleDbContext _db;

		public Repository(SampleDbContext db)
		{
			_db = db;
		}

		public TEntity? Get(int id)
		{
			return _db.Set<TEntity>().Find(id);
		}

		public IEnumerable<TEntity> All()
		{
			return _db.Set<TEntity>().AsEnumerable();
		}

		public IEnumerable<TEntity> All<TChild>(
			Expression<Func<TEntity, TChild>> includePredicate)
		{
			return _db.Set<TEntity>().Include(includePredicate).AsEnumerable();
		}

		public IEnumerable<TEntity> All<TChild, TGrandChild>(
			Expression<Func<TEntity, TChild>> includePredicate,
			Expression<Func<TChild, TGrandChild>> thenIncludePredicate)
		{
			return _db.Set<TEntity>().Include(includePredicate)
				.ThenInclude(thenIncludePredicate).AsEnumerable();
		}

		public TEntity? Select(Expression<Func<TEntity, bool>> predicate)
		{
			return _db.Set<TEntity>().FirstOrDefault(predicate);
		}

		public TEntity? Select<TChild>(Expression<Func<TEntity, bool>> predicate,
			Expression<Func<TEntity, TChild>> includePredicate)
		{
			return _db.Set<TEntity>().Include(includePredicate).FirstOrDefault(predicate);
		}

		public TEntity? Select<TChild, TGrandChild>(
			Expression<Func<TEntity, bool>> predicate,
			Expression<Func<TEntity, TChild>> includePredicate,
			Expression<Func<TChild, TGrandChild>> thenIncludePredicate)
		{
			return _db.Set<TEntity>().Include(includePredicate)
				.ThenInclude(thenIncludePredicate).FirstOrDefault(predicate);
		}

		public IEnumerable<TEntity> SelecyMany(Expression<Func<TEntity, bool>> predicate)
		{
			return _db.Set<TEntity>().Where(predicate).AsEnumerable();
		}

		public IEnumerable<TEntity> SelecyMany<TChild>(
			Expression<Func<TEntity, bool>> predicate,
			Expression<Func<TEntity, TChild>> includePredicate)
		{
			return _db.Set<TEntity>().Include(includePredicate).Where(predicate).AsEnumerable();
		}

		public IEnumerable<TEntity> SelecyMany<TChild, TGrandChild>(
			Expression<Func<TEntity, bool>> predicate,
			Expression<Func<TEntity, TChild>> includePredicate,
			Expression<Func<TChild, TGrandChild>> thenIncludePredicate)
		{
			return _db.Set<TEntity>().Include(includePredicate)
				.ThenInclude(thenIncludePredicate).Where(predicate).AsEnumerable();
		}

		public void Insert(TEntity entity)
		{
			_db.Set<TEntity>().Add(entity);
		}

		public void Insert(List<TEntity> entities)
		{
			_db.Set<TEntity>().AddRange(entities);
		}

		public void Update(TEntity entity)
		{
			_db.Set<TEntity>().Update(entity);
		}

		public void Update(List<TEntity> entities)
		{
			_db.Set<TEntity>().UpdateRange(entities);
		}

		public void Delete(TEntity entity)
		{
			_db.Set<TEntity>().Remove(entity);
		}

		public void Delete(List<TEntity> entities)
		{
			_db.Set<TEntity>().RemoveRange(entities);
		}

		public int SaveChanges()
		{
			return _db.SaveChanges();
		}
	}

	public interface IRepository<TEntity> where TEntity : class
	{
		TEntity? Get(int id);

		IEnumerable<TEntity> All();
		IEnumerable<TEntity> All<TChild>(Expression<Func<TEntity, TChild>> includePredicate);
		IEnumerable<TEntity> All<TChild, TGrandChild>(
			Expression<Func<TEntity, TChild>> includePredicate,
			Expression<Func<TChild, TGrandChild>> thenIncludePredicate);

		TEntity? Select(Expression<Func<TEntity, bool>> predicate);
		TEntity? Select<TChild>(
			Expression<Func<TEntity, bool>> predicate,
			Expression<Func<TEntity, TChild>> includePredicate);
		TEntity? Select<TChild, TGrandChild>(
			Expression<Func<TEntity, bool>> predicate,
			Expression<Func<TEntity, TChild>> includePredicate,
			Expression<Func<TChild, TGrandChild>> thenIncludePredicate);

		IEnumerable<TEntity> SelecyMany(Expression<Func<TEntity, bool>> predicate);
		IEnumerable<TEntity> SelecyMany<TChild>(
			Expression<Func<TEntity, bool>> predicate,
			Expression<Func<TEntity, TChild>> includePredicate);
		IEnumerable<TEntity> SelecyMany<TChild, TGrandChild>(
			Expression<Func<TEntity, bool>> predicate,
			Expression<Func<TEntity, TChild>> includePredicate,
			Expression<Func<TChild, TGrandChild>> thenIncludePredicate);

		void Insert(TEntity entity);
		void Insert(List<TEntity> entities);

		void Update(TEntity entity);
		void Update(List<TEntity> entities);

		void Delete(TEntity entity);
		void Delete(List<TEntity> entities);

		int SaveChanges();
	}
}

Serialize object/model to XML

string xml;
using (var writer = new StringWriter())
{
    var serializer = new XmlSerializer(typeof(MyModel));
    serializer.Serialize(writer, myModel);
    xml = writer.ToString();
}

Set default value of a GUID field in Entity Framework Sqlite vs MsSql


protected override void OnModelCreating(ModelBuilder modelBuilder)
{
   modelBuidler.Entity<MyEntity>(entity => 
      if (_settings.UseSql) 
      {
         entity.Property(e => e.Id)
            .HasColumnName("id")
            .HasDefaultValueSql("(newid())");
      }
      else if (_settings.UseSqlite)
      {
         entity.Property(e => e.Id)
            .HasColumnName("id")
            .HasDefaultValue(Guid.NewGuid());
      }
      ....
}

Convert a C# model into a list of SqlParameters

Assumes properties are decorated with [Column("column_name")] attributes.

public static List<SqlParameter> ToSqlParameters<T>(this T obj)
{
  var result = new List<SqlParameter>();
  var properties = obj.GetType().GetProperties();
    
  properties.ToList().ForEach(propertyInfo => 
  {
    var column = propertyInfo.GetCustomAttribute<ColumnAttribute>();
    var notMapped = propertyInfo.GetCustomAttribute<NotMappedAttribute>();

    // see previous post for this method "IsNullOrDefault"
    if (column != null && notMapped = null && !propertyInfo.IsNullOrDefault(obj)) 
    {
      var fieldValue = propertyInfo.GetValue(obj, default);
      var sqlParam = new SqlParameter($"@{column.Name}", fieldValue);

      result.Add(sqlParam);
    }
  });

  return result;
}

Convert a DataTable into a List of field/value pairs

public static List<Dictionary<string, dynamic>> ToDict(this DataTable dt) 
{
    var tableDict = new List<Dictionary<string, dynamic>>();

    foreach (DataRow row in dt.Rows)
    {
        var rowDict = new Dictionary<string, dynamic>();

        foreach (DataColumn col in dt.Columns) 
        {
            var fieldValue = row[col].Equals(DBNull.Value)
                ? null
                : row[col];

            rowDict.Add(col.ColumnName, fieldValue);
        }

        tableDict.Add(rowDict);
    }

    return tableDict;
}

Get enum definitions from C# API for Angular

    internal static class EnumExt
    {
        public static List<EnumDefinition> Define<TEnum>() where TEnum : Enum
        {
            var result = new List<EnumDefinition>();
            var keys = Enum.GetNames(typeof(TEnum));

            keys.ToList().ForEach(key =>
            {
                var desc = ((TEnum)Enum.Parse(typeof(TEnum), key))
                    .GetType()
                    .GetMember(key)
                    .FirstOrDefault()
                    ?.GetCustomAttribute<DescriptionAttribute>()
                    ?.Description;

                if (string.IsNullOrWhiteSpace(desc))
                {
                    desc = key;
                }

                var value = (int)(Enum.Parse(typeof(TEnum), key));

                var def = EnumDefinition.Create(key, value, desc);

                result.Add(def);
            });

            return result;
        }
    }
 internal class EnumDefinition
    {
        public string Key { get; private set; }
        public int Value { get; private set; }
        public string Description { get; private set; }

        public static EnumDefinition Create(
            string key, int value, string desc)
        {
            return new EnumDefinition
            {
                Key = key,
                Value = value,
                Description = desc
            };
        }
    }

Generic BLL and DAL for Reference Data with caching and backup files

BLL

internal class Blc<TEntity> : IBlc<TEntity>
{
    private readonly IDao<TEntity> _dao;
 
    public Blc(IDao<TEntity> dao)
    {
        _dao = dao;
    }
 
    /// <summary>
    /// <inheritdoc cref="IBlc{TEntity}.GetAsync()"/>
    /// </summary>
    public async Task<IEnumerable<TEntity>> GetAsync()
    {
        var cache = _dao.GetFromCache();
 
        if (cache != null && cache.Any())
        {
            return cache;
        }
 
        return await _dao.GetAsync();
    }
 
    /// <summary>
    /// <inheritdoc cref="IBlc{TEntity}.GetAsync(Expression{Func{TEntity, bool}})"/>
    /// </summary>
    public async Task<IEnumerable<TEntity>> GetAsync(Expression<Func<TEntity, bool>> expression)
    {
        var cache = _dao.GetFromCache();
 
        if (cache != null && cache.Any())
        {
            return cache.Where(expression);
        }
 
        return await _dao.GetAsync(expression);
    }
 
    /// <summary>
    /// <inheritdoc cref="IBlc{TEntity}.InitializeAsync()"/>
    /// </summary>
    public async Task InitializeAsync()
    {
        var results = await _dao.GetAsync();
 
        if (!results.Any())
        {
            var fileData = _dao.GetFromFile();
 
            if (fileData == null || !fileData.Any())
            {
                throw new KeyNotFoundException("Unable to get refdata from MongoDb nor from its backup file");
            }
 
            await _dao.InsertManyAsync(fileData);
        }
 
        _dao.InsertCache(results.ToList());
    }
}
 
public interface IBlc<TEntity>
{
    /// <summary>
    /// Gets a list of TEntity from MongoDb
    /// </summary>
    Task<IEnumerable<TEntity>> GetAsync();
 
    /// <summary>
    /// Gets a filtered list of TEntity
    /// </summary>
    /// <param name="expression">filter expression</param>
    /// <example>await _blc.GetAsync(x => x.Id == 1);</example>
    Task<IEnumerable<TEntity>> GetAsync(Expression<Func<TEntity, bool>> expression);
 
    /// <summary>
    /// Called during application StartUp.
    /// Checks if data already exists in database.
    /// If not, pulls backup data from a json file and inserts that into database,
    /// then inserts the data into the cache.
    /// </summary>
    /// <exception cref="KeyNotFoundException"></exception>
    Task InitializeAsync();
}

DAL

/// <summary>
/// Ideally, this class will not be called directly, but instead via Blc
/// </summary>
internal class Dao<TEntity> : IDao<TEntity> where TEntity : class
{
    private readonly IMemoryCache _cache;
    private readonly string _cacheKey;
    private readonly string _backupFile;
    private readonly IMongoCollection<TEntity> _mongoCollection;
    private readonly IJsonFileReader _fileReader;
 
    public Dao(
        MongoClient client,
        IOptionsMonitor<Dictionary<string, MongoCollectionConfiguration>> collectionConfigs,
        IOptionsMonitor<AppSettings> appSettings,
        IMemoryCache cache,
        IJsonFileReader fileReader)
    {
        BsonClassMap.RegisterClassMap<TEntity>(cm =>
        {
            cm.AutoMap();
            cm.SetIgnoreExtraElementsIsInherited(true);
            cm.SetIgnoreExtraElements(true);
        });
 
        var database = client.GetDatabase(appSettings.CurrentValue.DbName);
        var collectionConfig = collectionConfigs.CurrentValue.FirstOrDefault(x =>
            x.Key.ToLowerInvariant() == typeof(TEntity).Name.ToLowerInvariant()).Value;
 
        _cacheKey = collectionConfig.CacheKey;
        _backupFile = collectionConfig.BackupFileName;
        _mongoCollection = database.GetCollection<TEntity>(collectionConfig.CollectionName);
        _cache = cache;
        _fileReader = fileReader;
    }
 
    /// <summary>
    /// <inheritdoc cref="IDao{TEntity}.GetAsync()"/>
    /// </summary>
    public async Task<IEnumerable<TEntity>> GetAsync()
    {
        try
        {
            var cursorResult = await _mongoCollection.FindAsync(_ => true);
            return await cursorResult.ToListAsync();
        }
        catch (MongoException ex)
        {
            throw new DatabaseException(ex.ToString());
        }
    }
 
    /// <summary>
    /// <inheritdoc cref="IDao{TEntity}.GetAsync(Expression{Func{TEntity, bool}})"/>
    /// </summary>
    public async Task<IEnumerable<TEntity>> GetAsync(Expression<Func<TEntity, bool>> expression)
    {
        try
        {
            var filter = Builders<TEntity>.Filter.Where(expression);
            var cursorResult = await _mongoCollection.FindAsync(filter);
            return await cursorResult.ToListAsync();
        }
        catch (MongoException ex)
        {
            throw new DatabaseException(ex.ToString());
        }
    }
 
    /// <summary>
    /// <inheritdoc cref="IDao{TEntity}.InsertManyAsync(List{TEntity})"/>
    /// </summary>
    public async Task InsertManyAsync(List<TEntity> data)
    {
        try
        {
            await _mongoCollection.InsertManyAsync(data);
        }
        catch (MongoException ex)
        {
            throw new DatabaseInsertException(ex.ToString());
        }
    }
 
    /// <summary>
    /// <inheritdoc cref="IDao{TEntity}.GetFromFile"/>
    /// </summary>
    public List<TEntity> GetFromFile()
    {
        var json = _fileReader.ReadAllText(_backupFile);
        return JsonSerializer.Deserialize<List<TEntity>>(json);
    }
 
    /// <summary>
    /// <inheritdoc cref="IDao{TEntity}.GetFromCache"/>
    /// </summary>
    public IQueryable<TEntity> GetFromCache()
    {
        _cache.TryGetValue(_cacheKey, out IQueryable<TEntity> cacheResults);
        return cacheResults;
    }
 
    /// <summary>
    /// <inheritdoc cref="IDao{TEntity}.InsertCache(List{TEntity})"/>
    /// </summary>
    public void InsertCache(List<TEntity> data)
    {
        var cacheEntryOptions = new MemoryCacheEntryOptions()
            .SetSlidingExpiration(TimeSpan.FromHours(12));
 
        _cache.Set(_cacheKey, data, cacheEntryOptions);
    }
}
 
public interface IDao<TEntity>
{
    /// <summary>
    /// Pulls a list of TEntity from MongoDb.
    /// </summary>
    Task<IEnumerable<TEntity>> GetAsync();
 
    /// <summary>
    /// Pulls a filtered list of TEntity from MongoDb.
    /// Converts the given Linq expression to a Mongo filter.
    /// </summary>
    Task<IEnumerable<TEntity>> GetAsync(Expression<Func<TEntity, bool>> expression);
 
    /// <summary>
    /// Inserts many TEntity into MongoDb.
    /// </summary>
    /// <param name="data">the list to be inserted into MongoDb</param>
    Task InsertManyAsync(List<TEntity> data);
 
    /// <summary>
    /// Pulls a list of TEntity from its assigned backup json file.
    /// </summary>
    List<TEntity> GetFromFile();
 
    /// <summary>
    /// Pulls a list of TEntity from cache.
    /// </summary>
    IQueryable<TEntity> GetFromCache();
 
    /// <summary>
    /// Inserts a list of TEntity into cache
    /// </summary>
    void InsertCache(List<TEntity> data);
}

Startup.cs example for .NET Core API w/ Swagger

using Swashbuckle.AspNetCore.Swagger;

public void ConfigureServices(IServiceCollection services)
{
    services.AddMvcCore().AddApiExplorer();
    services.AddSwaggerGen();
    services.AddSwaggerGen(c =>
    {
        c.SwaggerDoc("v1", new Info { Title = "My API", Version = "v1" });
    });

    services.AddTransient<ISomeInterface, SomeDependency>();
}



// -----------------

public void Configure(IApplicationBuilder app, IHostingEnvironment env)
{
    if (env.IsDevelopment())
    {
        app.UseDeveloperExceptionPage();
        app.UseSwagger();
        app.UseSwaggerUI(c =>
        {
            c.SwaggerEndpoint("/swagger/v1/swagger.json", "My API");
        });
    }
    app.UseMvc();
}

How to mock IConfiguration

var configuration = new Mock<IConfiguration>();
 
var configurationSection = new Mock<IConfigurationSection>();
configurationSection.Setup(a => a.Value).Returns("testvalue");
 
configuration.Setup(a => a.GetSection("TestValueKey")).Returns(configurationSection.Object);   

Source